{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#先导入必要的工具包\n",
    "\n",
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#导入数据，并设置x和y的值\n",
    "\n",
    "train = pd.read_csv(\"RentListingInquries_FE_train.csv\")\n",
    "y_train = train['interest_level']\n",
    "\n",
    "train = train.drop([\"interest_level\"],axis = 1)\n",
    "x_train = train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': range(3, 10)}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#设置max_depth的值为3到10，步数为1，\n",
    "\n",
    "max_depth = range(3,10,1)\n",
    "\n",
    "param_test2 = dict(max_depth = max_depth)\n",
    "param_test2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\AI\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.60327, std: 0.00370, params: {'max_depth': 3},\n",
       "  mean: -0.59623, std: 0.00403, params: {'max_depth': 4},\n",
       "  mean: -0.59293, std: 0.00361, params: {'max_depth': 5},\n",
       "  mean: -0.59058, std: 0.00516, params: {'max_depth': 6},\n",
       "  mean: -0.59437, std: 0.00474, params: {'max_depth': 7},\n",
       "  mean: -0.59862, std: 0.00492, params: {'max_depth': 8},\n",
       "  mean: -0.60625, std: 0.00444, params: {'max_depth': 9}],\n",
       " {'max_depth': 6},\n",
       " -0.5905763459758381)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#把n_estimators的值替换为刚才得到的值220\n",
    "xgb2 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=220,  \n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "gsearch2 = GridSearchCV(xgb2, param_grid = param_test2, scoring='neg_log_loss',n_jobs=-1, cv=5)\n",
    "gsearch2.fit(x_train , y_train)\n",
    "\n",
    "gsearch2.grid_scores_, gsearch2.best_params_,     gsearch2.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'min_child_weight': range(3, 9)}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#得到最优的max_depth为6，设置min_child_weight的取值范围为3到9.步数为1\n",
    "\n",
    "min_child_weight = range(3,9,1)\n",
    "\n",
    "param_test3 = dict(min_child_weight=min_child_weight)\n",
    "param_test3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\AI\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.59125, std: 0.00568, params: {'min_child_weight': 3},\n",
       "  mean: -0.59104, std: 0.00447, params: {'min_child_weight': 4},\n",
       "  mean: -0.59076, std: 0.00448, params: {'min_child_weight': 5},\n",
       "  mean: -0.59189, std: 0.00459, params: {'min_child_weight': 6},\n",
       "  mean: -0.59088, std: 0.00481, params: {'min_child_weight': 7},\n",
       "  mean: -0.59203, std: 0.00453, params: {'min_child_weight': 8}],\n",
       " {'min_child_weight': 5},\n",
       " -0.5907647911322124)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#把n_estimators和max_depth分别换成刚才得到的最优参数220和6\n",
    "xgb3 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=220,  \n",
    "        max_depth=6,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "gsearch3 = GridSearchCV(xgb3, param_grid = param_test3, scoring='neg_log_loss',n_jobs=-1, cv=5)\n",
    "gsearch3.fit(x_train , y_train)\n",
    "\n",
    "gsearch3.grid_scores_, gsearch3.best_params_,     gsearch3.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'subsample': [0.3, 0.4, 0.5, 0.6, 0.7, 0.8],\n",
       " 'colsample_bytree': [0.6, 0.7, 0.8, 0.9]}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#得到min_child_weight的的最优参数为5，接下来调整subsample和colsample_bytree的参数\n",
    "subsample = [i/10.0 for i in range(3,9)]\n",
    "colsample_bytree = [i/10.0 for i in range(6,10)]\n",
    "param_test4 = dict(subsample=subsample, colsample_bytree=colsample_bytree)\n",
    "param_test4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\AI\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.59091, std: 0.00441, params: {'colsample_bytree': 0.6, 'subsample': 0.3},\n",
       "  mean: -0.58945, std: 0.00335, params: {'colsample_bytree': 0.6, 'subsample': 0.4},\n",
       "  mean: -0.58705, std: 0.00493, params: {'colsample_bytree': 0.6, 'subsample': 0.5},\n",
       "  mean: -0.58564, std: 0.00379, params: {'colsample_bytree': 0.6, 'subsample': 0.6},\n",
       "  mean: -0.58527, std: 0.00437, params: {'colsample_bytree': 0.6, 'subsample': 0.7},\n",
       "  mean: -0.58502, std: 0.00387, params: {'colsample_bytree': 0.6, 'subsample': 0.8},\n",
       "  mean: -0.59126, std: 0.00448, params: {'colsample_bytree': 0.7, 'subsample': 0.3},\n",
       "  mean: -0.58870, std: 0.00473, params: {'colsample_bytree': 0.7, 'subsample': 0.4},\n",
       "  mean: -0.58737, std: 0.00373, params: {'colsample_bytree': 0.7, 'subsample': 0.5},\n",
       "  mean: -0.58533, std: 0.00408, params: {'colsample_bytree': 0.7, 'subsample': 0.6},\n",
       "  mean: -0.58544, std: 0.00365, params: {'colsample_bytree': 0.7, 'subsample': 0.7},\n",
       "  mean: -0.58512, std: 0.00446, params: {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       "  mean: -0.59076, std: 0.00448, params: {'colsample_bytree': 0.8, 'subsample': 0.3},\n",
       "  mean: -0.58874, std: 0.00460, params: {'colsample_bytree': 0.8, 'subsample': 0.4},\n",
       "  mean: -0.58767, std: 0.00393, params: {'colsample_bytree': 0.8, 'subsample': 0.5},\n",
       "  mean: -0.58539, std: 0.00413, params: {'colsample_bytree': 0.8, 'subsample': 0.6},\n",
       "  mean: -0.58425, std: 0.00399, params: {'colsample_bytree': 0.8, 'subsample': 0.7},\n",
       "  mean: -0.58433, std: 0.00294, params: {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       "  mean: -0.59186, std: 0.00462, params: {'colsample_bytree': 0.9, 'subsample': 0.3},\n",
       "  mean: -0.58835, std: 0.00372, params: {'colsample_bytree': 0.9, 'subsample': 0.4},\n",
       "  mean: -0.58686, std: 0.00385, params: {'colsample_bytree': 0.9, 'subsample': 0.5},\n",
       "  mean: -0.58549, std: 0.00283, params: {'colsample_bytree': 0.9, 'subsample': 0.6},\n",
       "  mean: -0.58562, std: 0.00385, params: {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       "  mean: -0.58464, std: 0.00395, params: {'colsample_bytree': 0.9, 'subsample': 0.8}],\n",
       " {'colsample_bytree': 0.8, 'subsample': 0.7},\n",
       " -0.5842528447204194)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#把之前得到的n_estimators=220；max_depth=6；min_child_weight=5分别带入\n",
    "xgb4 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=220,  \n",
    "        max_depth=6,\n",
    "        min_child_weight=5,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "gsearch4 = GridSearchCV(xgb4, param_grid = param_test4, scoring='neg_log_loss',n_jobs=-1, cv=5)\n",
    "gsearch4.fit(x_train , y_train)\n",
    "\n",
    "gsearch4.grid_scores_, gsearch4.best_params_,     gsearch4.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.584253 using {'colsample_bytree': 0.8, 'subsample': 0.7}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\AI\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\AI\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\AI\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\AI\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\AI\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\AI\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\AI\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xdc1dX/wPHXYYsoKkNwg5k5QNyaE8m9cmS5UbFhpn4rv2qWmmVZ+TNn+XXizJnmypF7b1xggGiCE1QcKMp4//64VwIBReQq4Hk+HvcR937OOZ/zuaFvP+ecz/soEUHTNE3TMsvsZXdA0zRNy9l0INE0TdOeiw4kmqZp2nPRgUTTNE17LjqQaJqmac9FBxJN0zTtuehAommapj0XHUg0TdO056IDiaZpmvZcLF52B14ER0dHKVWq1MvuhqZpWo5y5MiRKBFxelq5VyKQlCpVisOHD7/sbmiapuUoSql/MlJOD21pmqZpz8WkgUQp1Uwp9bdSKlQpNTSN475KqUilVIDx5Zfs2A9KqVPG17vJPl9obPOUUmq2UsrSlNegaZqmPZnJAolSyhyYCjQHygOdlVLl0yi6RES8jK+ZxrotgSqAF1ATGKyUym8svxB4A/AA8gB+abSpaZqmvSCmnCOpAYSKSBiAUmox0BYIzEDd8sAOEYkH4pVSx4FmwFIRWf+okFLqIFAsy3uuaSYQFxdHREQEsbGxL7srmpaCjY0NxYoVw9IycwM8pgwkRYHwZO8jMNxdPK6DUqo+EAz8R0TCgePASKXUeMAW8OaxAGQc0uoODDRB3zUty0VERJAvXz5KlSqFUupld0fTABARrl+/TkREBG5ubplqw5RzJGn9SXl8F601QCkR8QT+AuYCiMgmYD2wF/gN2AfEP1b3F2CniOxK8+RKva+UOqyUOhwZGZn5q9C0LBIbG4uDg4MOIlq2opTCwcHhue6UTRlIIoDiyd4XAy4lLyAi10XkgfHtDKBqsmNjjPMmjTEEpZBHx5RSIwEn4NP0Ti4i00WkmohUc3J66jJoTXshdBDRsqPn/b00ZSA5BJRRSrkppayA94DVyQsopVyTvW0DBBk/N1dKORh/9gQ8gU3G935AU6CziCSasP/c2bqNm8uWmfIUmqZpOZ7J5khEJF4p1R/YCJgDs0XktFJqNHBYRFYDA5RSbTAMW90AfI3VLYFdxih5G+hmnHgHmAb8A+wzHv9dREab4hqiV6wgZvdubKtWw9o9c2OHmqZpuZ1JnyMRkfUi8rqIlBaRMcbPRhiDCCIyTEQqiEglEfEWkTPGz2NFpLzxVUtEApK1aWFs79GSYZMEEQCXkSNQNjZc/uILJCHBVKfRtBcmOjqaX375JVN1J0yYwL1797KsL/7+/vTv3z/L2suo7du306pVq2eu96z9PX/+PIsWLXrm82TUjRs3aNy4MWXKlKFx48bcvHkzzXIXLlygSZMmlCtXjvLly3P+/Pks74t+sv0JLJ2dcflyOPcDArgxd97L7o6mPbfsFEhyuycFkvj4x9cOPbuxY8fi4+NDSEgIPj4+jB07Ns1yPXr0YPDgwQQFBXHw4EGcnZ2f+9yPeyVybT2P/K1acfvPDUROnIhdw4Z6iEvLEl+vOU3gpdtZ2mb5IvkZ2brCE8sMHTqUs2fP4uXlRePGjXF2dmbp0qU8ePCAdu3a8fXXXxMTE0OnTp2IiIggISGBr776iqtXr3Lp0iW8vb1xdHRk27Ztaba/YcMGvvjiCxISEnB0dGTLli3cuHGD3r17ExYWhq2tLdOnT8fT0zNFvWXLlvH1119jbm6Ovb09O3fu5Pz583Tv3p2YmBgApkyZwptvvsn27dsZOXIkhQsXJiAggPbt2+Ph4cHEiRO5f/8+q1atonTp0vj6+mJjY8Pp06e5evUq48ePT3UnEhMTwyeffMLJkyeJj49n1KhRtG3bNt3vLzw8nGbNmnHu3Dm6dOnCyJEj+eqrr3B0dGTgQMOTCMOHD6dw4cIsWrSIoKAgvLy86NmzJwULFmTdunXExsYSExPD1q1b+emnn1J9/wALFixg0qRJPHz4kJo1a/LLL79gbm6eoi9//PEH27dvB6Bnz540bNiQH374IUWZwMBA4uPjady4MQB2dnZP+vXINB1InkIphcuokYS1bsPlL76g5MIFqMf+h2paTjF27FhOnTpFQEAAmzZtYvny5Rw8eBARoU2bNuzcuZPIyEiKFCnCunXrALh16xb29vaMHz+ebdu24ejomGbbkZGR9O3bl507d+Lm5saNGzcAGDlyJJUrV2bVqlVs3bqVHj16EBAQkKLu6NGj2bhxI0WLFiU6OhoAZ2dnNm/ejI2NDSEhIXTu3Dkp+erx48cJCgqiUKFCuLu74+fnx8GDB5k4cSKTJ09mwoQJgOGuYMeOHZw9exZvb29CQ0NTnHfMmDE0atSI2bNnEx0dTY0aNXjrrbfImzdvmtd48OBBTp06ha2tLdWrV6dly5b06dOH9u3bM3DgQBITE1m8eDEHDx7E09OTcePGsXbtWsAwNLZv3z5OnDhBoUKF2LRpEyEhIam+fycnJ5YsWcKePXuwtLSkX79+LFy4kB49euDn58eHH35ItWrVuHr1Kq6uhvVKrq6uXLt2LVV/g4ODKVCgAO3bt+fcuXO89dZbjB07NlVQel46kGTAoyGuS4P/y42583Do3etld0nL4Z525/AibNq0iU2bNlG5cmUA7t69S0hICPXq1ePzzz9nyJAhtGrVinr16mWovf3791O/fv2kh9oKFSoEwO7du1mxYgUAjRo14vr169y6dStF3Tp16uDr60unTp1o3749YMgE0L9/fwICAjA3Nyc4ODipfPXq1ZP+Ei1dujRNmjQBwMPDI8XdUqdOnTAzM6NMmTK4u7tz5syZVN/B6tWrGTduHGB41ufChQuUK1cuzWts3LgxDg4OALRv357du3czaNAgHBwcOHbsGFevXqVy5cpJZdKq/+h7Se/7P3HiBEeOHKF69eoA3L9/P2k4aubMmWm2m574+Hh27drFsWPHKFGiBO+++y7+/v706dPnmdp5Gh1IMkgPcWm5jYgwbNgwPvjgg1THjhw5wvr16xk2bBhNmjRhxIgRGWovrecRRB5/Djn1cwvTpk3jwIEDrFu3Di8vLwICApg8eTKFCxfm+PHjJCYmYmNjk1Te2to66WczM7Ok92ZmZinmHx4/z+PvRYQVK1ZQtmzZp17fk9rz8/PD39+fK1eu0Lt373TrJ7/TSe/7nzx5Mj179uT7779/Yl8KFy7M5cuXcXV15fLly2nOfRQrVozKlSvj7u4OwNtvv83+/fuzPJDoyfYMejTEpVdxaTlZvnz5uHPnDgBNmzZl9uzZ3L17F4CLFy9y7do1Ll26hK2tLd26dePzzz/n6NGjqeqmpXbt2uzYsYNz584BJA1t1a9fn4ULFwKGFVOOjo7kz58/Rd2zZ89Ss2ZNRo8ejaOjI+Hh4dy6dQtXV1fMzMyYP38+CZn4M7ds2TISExM5e/YsYWFhqQJG06ZNmTx5clKwO3bs2BPb27x5Mzdu3Eiai6lTpw4A7dq1Y8OGDRw6dIimTZsCT/++0vv+fXx8WL58edJQ1Y0bN/jnn9TbgrRp04a5c+cCMHfu3DTndqpXr87Nmzd5lN1j69atlC+fVu7c56PvSJ6BHuLScjoHBwfq1KlDxYoVad68OV26dKF27dqAYSJ2wYIFhIaGMnjwYMzMzLC0tOTXX38F4P3336d58+a4urqmOdnu5OTE9OnTad++PYmJiUlzHKNGjaJXr154enpia2ub9JdfcoMHDyYkJAQRwcfHh0qVKtGvXz86dOjAsmXL8Pb2Tnfe4knKli1LgwYNuHr1KtOmTUtxVwPw1VdfMWjQIDw9PRERSpUqlTSnkZa6devSvXt3QkND6dKlC9WqVQPAysoKb29vChQokDT/4OnpiYWFBZUqVcLX15eCBQumaKtJkyYEBQWl+v7Lly/Pt99+S5MmTUhMTMTS0pKpU6dSsmTJFHMkQ4cOpVOnTsyaNYsSJUqwzPjw9OHDh5k2bRozZ87E3NyccePG4ePjg4hQtWpV+vbt+8zf49OotG47c5tq1apJVu2QKCJEfNyfmD17cFu5Ug9xaRkWFBSU7ti7lvV8fX1p1aoVHTt2NPm5EhMTqVKlCsuWLaNMmTImP58ppPX7qZQ6IiLVnlZXD209Iz3EpWlacoGBgbz22mv4+Pjk2CDyvPTQViboIS7tVVezZk0ePHiQ4rP58+fj4eHxknqUmr+/f6bqbdy4kSFDhqT4zM3NjZUrV6ZZvnz58oSFhWXqXLmFDiSZpFdxaa+yAwcOvOwumEzTpk2TJsy1jNFDW5mkh7g0TdMMdCB5EhGIf5juYZ2LS9M0TQeS9InA6v6wvBckpJ9gLX+rVtg1akTkxIk8CDv3AjuoaZqWPehAkh6lwKUSnFkLawcZAkuaxfQQl6ZprzYdSJ6k5vvQYAgcmw9bvk63mB7i0nKK7JRGXu9H8nwysh/Jtm3b8PLySnrZ2NiwatWqLO+LDiRP03AYVOsNu3+GvVPSLaaHuLScIDsFktwuO+xH4u3tTUBAAAEBAWzduhVbW9ukBJdZyaTLf5VSzYCJGLbanSkiYx877gv8BFw0fjRFRGYaj/0AtDR+/o2ILDF+3h8YBJQGnEQkypTXgFLQYhzcuw6bhoOtA3h1TqOYTjevPYM/h8KVk1nbposHNE97c6NH9H4kr9Z+JMktX76c5s2bY2trm26ZTBMRk7wwBI+zgDtgBRwHyj9WxhdD8Hi8bktgM4ZAlxc4DOQ3HqsMlALOA44Z6UvVqlXlucXFivi3FhlVUOTvDekWi169WgLLviFRs2Y//zm1XCUwMPDfN+uHiMxukbWv9UOe2odz585JhQoVRERk48aN0rdvX0lMTJSEhARp2bKl7NixQ5YvXy5+fn5JdaKjo0VEpGTJkhIZGZlu29euXZNixYpJWFiYiIhcv35dRET69+8vo0aNEhGRLVu2SKVKlUREZM6cOfLxxx+LiEjFihUlIiJCRERu3rwpIiIxMTFy//59EREJDg6WR3+Ot23bJvb29nLp0iWJjY2VIkWKyIgRI0REZMKECTJw4EAREenZs6c0bdpUEhISJDg4WIoWLSr379+Xbdu2ScuWLUVEZNiwYTJ//vyk85YpU0bu3r2b5vXNmTNHXFxcJCoqSu7duycVKlSQQ4cOyblz56Ry5coiIpKQkCDu7u4SFRWV4jyP6hctWjTpe0nv+w8MDJRWrVrJw4cPRUTko48+krlz54qISJ8+feTQoUMiImJvb5+ifwUKFEj3/42IiLe3t6xZsybd4yl+P42Aw5KBv2NNeUdSAwgVkTAApdRioC0QmIG65YEdIhIPxCuljgPNgKUicszYnml6nR4La3hvIcxtDUt7Qo9VUKJWqmL6QUUtQ55y5/Ai6P1Icv9+JI9cvnyZkydPmuxBS1MGkqJAeLL3EUDNNMp1UErVB4KB/4hIOIa7l5FKqfGALeBNxgKQaVnng67LYXZTWNQJev0JhVNuUKSHuLScQvR+JLl+P5JHli5dSrt27bC0tHxim5llysn2tG4ZHv+NWgOUEhFP4C9gLoCIbALWA3uB34B9wDPNTiml3ldKHVZKHX6Uiz9L5HWE7ivBMi/Mbw83z6cqoldxadmV3o/k1dqP5JHffvuNzp1Tz+1mFVMGkgigeLL3xYBLyQuIyHUReZT5bQZQNdmxMSLiJSKNMQSlkGc5uYhMF5FqIlLNyckpUxcwe/c5Jm0J4UH8Y7/ABUpA998hPhbmt4O7qQOVXsWlZUfJ9yPZvHlz0n4kHh4edOzYkTt37nDy5Elq1KiBl5cXY8aM4csvvwT+3Y/E29s7zbaT70dSqVIl3n33XQBGjRrF4cOH8fT0ZOjQoenuR+Lh4UHFihWpX79+0n4kc+fOpVatWgQHBz/XfiTNmzdPdz+SuLg4PD09qVixIl999dUT23u0H4mXlxcdOnRItR9Jp06d0tyP5Oeff07VVpMmTdL8/pPvR+Lp6Unjxo25fPkyYLjzebQlxtChQ9m8eTNlypRh8+bNDB06FDDsR+Ln55d0nvPnzxMeHk6DBg2e+fvLsIxMpGTmhWHYLAxw49/J9gqPlXFN9nM7YL/8O1HvYPzZEzgFWDxW9zwmnmwfvCxASg5ZK97jtsne0KjUBS4cEPmmsMivdUXu30p1+OHVq3KmRk059+57khgfn6k+aLlHWpOZmun07NlTli1b9kLOlZCQIJUqVZLg4OAXcj5TeJ7JdpPdkYhhorw/sBEIwjBRflopNVop1cZYbIBS6rRxMn0AhlVcAJbALqVUIDAd6GZsD6XUAKVUBIY7nBNKqczNPmXAjx0r4d+rOnEJiXSesZ/Plh7nRkyy3FvFa8C78+FaICzuAnGxKerrIS5Ny/30fiR6h8QniomLQaFQYs3krSFM3xlGPhsLhrUoxztVi/078XZiKfzeF8q1hnfmgtm/k+uid1TUjHLTDok5YT+SzHrW/Uhyi+fZIVEHknSICAO2DiDyfiRTfKbgmMeRv6/cYfjKkxz+5yY13Qoxpp0HrznbGSrs/xU2DIUqPaH1RMODjEZx164R1roN1m5uehXXKyw3BRIt99Fb7ZqAUooOr3cg7FYYXdZ1IfRmKGVd8rH0g9qMbe/BmSt3aD5xJ+M3/U1sXALU+gjqfQ5H58LWb1O0pYe4NE3LzXQgeYKGxRsyp9kc4hLj6P5nd/Zd2oeZmeK9GiXY8lkDWnkWYdLWUJpN2MnukCho9KXhjmTXOMMdSjJ6FZemabmVDiRPUcGhAotaLMLVzpV+f/VjZYhhnNTRzpqf3/ViQR/DM5bdZh1g0JIAohqONcyVbBhqmDsx0unmNU3LrXQgyQBXO1fmNZtHDdcajNg7golHJ5IoiQDULePIhkH1GeBThnUnL9No/C6WlBiJlKoHqz6CkM1J7eghLu1ly07Zf3Ua+eeTkTTyAP/973+pUKEC5cqVY8CAAWlmGnheOpBkkJ2VHVN8ptChTAdmnpzJkJ1DeJBgWLViY2nOp41f58+B9Snnmp8hfwTTI2YgsYXKwZLuEH4wqR09xKW9TNkpkOR22SGN/N69e9mzZw8nTpzg1KlTHDp0iB07djz3uR+nA8kzsDSzZGTtkQyqMogN5zfgt9GPm7H//ivgNWc7Fr9fi586enLqulD/0sfcsHBAFr4D14IAPcSlvVzJ08gPHjyYn376ierVq+Pp6cnIkSMBQ2r1li1bUqlSJSpWrMiSJUuYNGlSUhr59J5sB0Ma+SpVqlCpUiV8fHwAw7+c3377bTw9PalVqxYnTpxIVW/ZsmVUrFiRSpUqUb9+fcDwF3G9evWoUqUKVapUYe/evYDhjqJBgwZ06tSJ119/naFDh7Jw4UJq1KiBh4cHZ8+eBcDX15cPP/yQevXq8frrr7N27dpU542JiaF3795Ur16dypUr88cffzzx+3uURr5s2bJJKd+/+uorJk6cmFRm+PDhTJo0iaFDh7Jr1y68vLz4+eef8ff355133qF169ZJSSbT+v7BkEb+UXaBDz74IM30MH/88Qc9e/YEDGnk09qwSilFbGwsDx8+5MGDB8TFxVG4cOEnXmNmmHQ/ktxIKUUfjz4Uy1eML3Z9Qdf1XfnF5xdK2ZdKOv5OteL4lCvMd+uDaHP0c1bZfI3dnLbYfPAXFCiRNMR1afB/uTF3Hg69e73ci9JeuB8O/sCZG2eeXvAZvFHoDYbUGPLEMmPHjuXUqVMEBASwadMmli9fzsGDBxER2rRpw86dO4mMjKRIkSKsW7cOgFu3bmFvb8/48ePZtm0bjo6OabYdGRlJ37592blzJ25ubkm5tkaOHEnlypVZtWoVW7dupUePHgQEBKSoO3r0aDZu3EjRokWJjo4GwNnZmc2bN2NjY0NISAidO3dOSg9y/PhxgoKCKFSoEO7u7vj5+XHw4EEmTpzI5MmTmTBhAmAIRjt27ODs2bN4e3sTGhqa4rxjxoyhUaNGzJ49m+joaGrUqMFbb72VbjqWgwcPcurUKWxtbalevTotW7akT58+tG/fnoEDB5KYmMjixYs5ePAgnp6ejBs3LimA+fv7s2/fPk6cOEGhQoXYtGkTISEhqb5/JycnlixZwp49e7C0tKRfv34sXLiQHj164Ofnx4cffki1atW4evVqUgZkV1fXpNxcydWuXRtvb29cXV0REfr372+SJej6jiSTmpZqyqyms7j78C7d/uzGkatHUhwvlNeKce9UYlzfNgzNM4oH9+5wdWoLIq9EAHqIS3v5kqcxr1KlCmfOnCEkJAQPDw/++usvhgwZwq5du7C3t89Qe09KI9+9e3fg6WnkZ8yYkfSv77i4OPr27YuHhwfvvPMOgYH/JgB/lEbe2to6VRr58+fPJ5XLSBr5sWPH4uXlRcOGDZPSyKfnURr5PHnyJKWRL1WqVFIa+Uff57OmkU/+/W/ZsiUpjbyXlxdbtmwhLCwMMKSRf5TfKyNCQ0MJCgoiIiKCixcvsnXrVnbu3Jnh+hml70ieg5ezFwtbLKTfln703dSX0XVG08o95SReLXcHKn/anTWr89Lq+EeE/tqGv96aT6c65XW6+VfY0+4cXgTRaeRzfRr5lStXUqtWLezsDA9ON2/ePCngZyV9R/KciucvzoIWC6jkVIlhu4bxv+P/S/UHx9rCnI7t3yG69SzKqXMU39SXTr/sIDjeWq/i0l4onUb+1UojX6JECXbs2EF8fDxxcXHs2LHDJENb+o4kC9hb2/O/xv9j1N5RTAmYQvidcEbWHomlecpNZFyqtUUsplJ31UfEXv+et6d8gu+b7nRt2FDvqKi9EMnTyDdv3jwpjTmAnZ0dCxYsIDQ0lMGDB2NmZoalpSW//mp4uPZRGnlXV9c092xPnkY+MTExaY5j1KhR9OrVC09PT2xtbdNNIx8SEoKI4OPjk5RGvkOHDixbtgxvb+/nSiN/9erVdNPIDxo0CE9PT0SEUqVKpTkp/8ijNPKhoaF06dIlVRr5AgUKpJlG3tfXl4IFC6Zoq0mTJgQFBaX6/pOnkU9MTMTS0pKpU6dSsmTJFHMkQ4cOpVOnTsyaNYsSJUqwbNkywJBGftq0acycOZOOHTuydetWPDw8UErRrFkzWrdu/czf49PoXFtZSESYdnwavxz/hZouNRnvPZ78VvlTF9w7BTYNZ3+htrx3qRPlreL4af335H2ttB7iysV0rq0Xy9fXl1atWtGxY0eTnysxMZEqVaqwbNmyHJsBWOfayiaUUnzk9RHf1f2OI9eO0H19dyLuRKQu+GZ/qPsfat34g701DxBfsBDjyrbmfkAA5/8368V3XNO0TNNp5PXQlkm0Lt0al7wuDNw2kK7ruzK50WQ8nTxTFvIZCTGRFDk2ifVNizLd6z0OXj5OpalTWORcjnfb18XcLK3dijXt5csJaeT9/f0zVe9Z08iXL18+aVXVq0oPbZlQ2K0w+v3Vj6j7UYytN5a3Sr6VskBCPCzrCWfWQodZnLeqzo13O3DO1hH/d4fxXQcvPIplbOmllv3poS0tO9NDW9mUu707C1sspGyhsny6/VPmnp6bckWXuQV0mAUl68LKDyhlHoL76BGUv/EPXgc30HbqbkatPs2d2LiXdxGapmlPYdJAopRqppT6WykVqpQamsZxX6VUpFIqwPjyS3bsB6XUKePr3WSfuymlDiilQpRSS5RSVqa8huflkMeBWU1m8VbJtxh3eBzf7v+W+MRkeXYsbaDzInA25OWyr+yKXaNGdDn1Jx+6WzB333kaj9/JhlOXTZJsTdM07XmZLJAopcyBqUBzoDzQWSlVPo2iS0TEy/iaaazbEqgCeAE1gcFKqUfLn34AfhaRMsBNoI+priGr2FjYMK7BOHpV7MXS4KV8svUTYuJikhWwh26/g11h1KJ3cPmkG2Y2NnTa4s/v79ekYF4rPlxwlL7zDhNxUyfN0zQtezHlHUkNIFREwkTkIbAYSP3ETNrKAztEJF5EYoDjQDNleIy0EbDcWG4u8HYW99skzJQZn1b9lBG1R7Dv0j56/NmDKzFX/i1g5wzdV4K5FZZ/9sFl0AfcDwigxLY1rOlfh+EtyrEn9DqNx+9kxs4w4hMSX97FaJqmJWPKQFIUCE/2PsL42eM6KKVOKKWWK6WKGz87DjRXStkqpRwBb6A44ABEi8ijsaH02sy23nn9Hab6TOXi3Yt0Xdc1ZeK+Qm7QbQU8uEv+qKnYNahL5MSJJPzzD33ru7P50/rUec2BMeuDaD1lD8cupL3/gKalJzulkdf7kTyfjO5HMmTIECpWrJiUydkUTBlI0lq7+vgg/xqglIh4An9huMNARDYB64G9wG/APiA+g20aTq7U+0qpw0qpw5GRkZm7AhOpU7QOc5vNRSlFjz97sDMiWRI1Fw/oshh1KxyXN4JR1tZJ6eaLFbRlRo9qTOtWlZsxD2n/616+WnWK23oyXsug7BRIcrvssB/JunXrOHr0KAEBARw4cICffvqJ27dvP/e5H2fKQBKB4S7ikWLApeQFROS6iDxajD4DqJrs2BjjvEljDAEkBIgCCiilLNJrM1n96SJSTUSqOTk5ZckFZaWyhcqyqOUiSuUvxSdbP2HxmcX/Hiz5JnScg+Xtk7g0zJsiF5dSimYVXfjrswb4vlmKhQf+wef/drD2xCU9Ga89ld6PJKXcvh9JYGAgDRo0wMLCgrx581KpUiU2bNjwxGvMFBExyQvDw45hgBtghWG4qsJjZVyT/dwO2G/82RxwMP7sCZwCLIzvlwHvGX+eBvR7Wl+qVq0q2VXMwxjp/1d/qehfUX48+KPEJ8T/e/DoAkkckV8utK0tQZ6VJPZsWKr6J8KjpdWkXVJyyFrpMeuA/BMV8wJ7rz2LwMDApJ8vjxkj57t1z9LX5TFjntqHc+fOSYUKFUREZOPGjdK3b19JTEyUhIQEadmypezYsUOWL18ufn5+SXWio6NFRKRkyZISGRmZbtvXrl2TYsWKSViY4ff0+vXrIiLSv39/GTVqlIiIbNmyRSpVqiQiInPmzJGPP/5YREQqVqwoERERIiJy8+ZNERGJiYmR+/c3lzzqAAAgAElEQVTvi4hIcHCwPPpzvG3bNrG3t5dLly5JbGysFClSREaMGCEiIhMmTJCBAweKiEjPnj2ladOmkpCQIMHBwVK0aFG5f/++bNu2TVq2bCkiIsOGDZP58+cnnbdMmTJy9+7dNK9vzpw54uLiIlFRUXLv3j2pUKGCHDp0SM6dOyeVK1cWEZGEhARxd3eXqKioFOd5VL9o0aJJ30t6339gYKC0atVKHj58KCIiH330kcydO1dERPr06SOHDh0SERF7e/sU/StQoECqPm/cuFHefPNNiYmJkcjISHFzc5Nx48aleX3Jfz8fAQ5LBv6+N9mT7SISr5TqD2w0BobZInJaKTXa2LnVwAClVBsMw1Y3AF9jdUtglzFF822gm/w7LzIEWKyU+hY4BuTonCK2lrZM8J7Aj4d+ZF7gPC7evcj39b4nj0UeqNwVdS8Kl9hRhP1TPM108x7F7Fn1cR3m7TvPuI1/0/jnHQx8qwx967ljaa4fE9LSl3w/DIC7d+8SEhJCvXr1+PzzzxkyZAitWrWiXr16GWrvSfuRrFixAnj6fiSdOnWiffv2gGE/kv79+xMQEIC5uTnBwcFJ5R/tRwKk2o8keULJjOxHsnr1asaNGweQtB9Jeg+OPtqPBEjaj2TQoEFJ+5FcvXo1U/uRwL/f/4kTJ5L2IwG4f/9+Uor4mTNnptluepo0acKhQ4d48803cXJyonbt2lhYZP1f+yZNkSIi6zHMdST/bESyn4cBw9KoF4th5VZabYZhWBGWa5ibmTOs5jBK5C/BDwd/oPeG3kz2mYxjHkeoMxDLmEhcrs7k0v64NHdUNDdT9KrjRrOKLny9OpAfN/zNqmMX+a6dB9VKFXpJV6U9icsXX7zsLuj9SMj9+5GAYaht+PDhAHTp0sUk+cD0P1mzka7lujLReyJnb52l67quhN40bgva+Bvyt22HXdH7RP48Pt0dFV3t8zCte1Vm9qhGzIMEOk7bx7DfTxB97+ELvAotO9P7kbxa+5EkJCRw/fp1AE6cOMGJEyeS7t6ykk7amM14l/BmTtM59N/anx5/9mC893hqudZCtZmMy7V3CZt6isuDPqTkyvXpppt/q3xhapd2YOKWEGbtPsem01f5qlV52noVSfNfjNqrQ+9H8mrtRxIXF5c0NJk/f34WLFhgkqEtnbQxm7p09xIfb/mY87fOM6L2CNqVaQdx97k1tDGX1l3HuU97HAaPeWo7gZdu88XKkwSER1P3NUe+ebsibo7P/gdSe346aeOLpfcjeTY6aWMuVMSuCPOaz6O6S3VG7B3BpKOTSLSwJv83a7BztyTSfwUP9q95ajvli+RnxUdv8k3bChwPj6bphJ1M2hLCg/hnHybQNC01vR+JviPJ9uIS4xizfwwrQlbQvFRzvqn7DWYXQgl7uyPW+RMpuWwVyiVj/8q9djuW0WsDWXviMqWd8jKmnQe13NNeXaJlvdx0R5IT9iPJrGfdjyS3eJ47Eh1IcgARYdapWUw8OpEqzlWY4D0Bs99XcGn0/+FcS+EwcTPYZzxTzPa/r/HVH6cIv3Gfd6oWY1iLchTKm62TKOcKuSmQaLmPHtrK5ZRS+Hn48VODnzgVdYpu67txs+Vb2L1ZlchDiTyY1Bbu3chwew3LOrNpUAP6NSzNymMX8fm/7Sw7HK6fjH8B9HesZUfP+3upA0kO0qxUM2Y1ncWdh3fo9mc3Ij/vhcqTl8sbo5EF78DDmKc3YpTHypz/NnuDdQPqUdrJjsHLT/De9P2EXrtrwit4tdnY2HD9+nUdTLRsRUS4fv16qhVtz0IPbeVA4bfD6belHxfvXuTnB+1wHrcIZ6/bOLSqBe/9BhbPNkyVmCgsORzO9+uDuB+XwLDm5ehd181EvX91xcXFERERQWxs7MvuiqalYGNjQ7FixbC0tEzxuZ4jSSa3BRKAWw9uMXDbQI5cOcyvf5XE8fgF3BpfxLpOe2g3Hcye/WYz6u4Dhv1+ks2BV/myZTn86rmboOeapuUUeo4kl7O3tmd64+m0LN2KYbX/IdZScemMB3J8GWwcBpn4B4KjnTW/dq1C84oufLsuiLl7z2d9xzVNy3V0IMnBrMyt+L7u97xX5yOm+yQQG3aNS7E+cGAa7BqXqTYtzM2Y1LkyjcsXZuTq0yw8kDo1g6ZpWnI6kORwSik+9vqYpu+P4XAZM66vP8OFwk1h67dweHam2rQ0N2NKl8p4l3Vi+MpTLD0U/vRKmqa9snQgySXalnmbN77/mYcWcOL3IE6614O1n8Lp1JvdZIS1hTm/dqtKvTKODPn9BCuPRWRxjzVNyy10IMlFqldsgv2QTykdHs/Sg+FsKeEJv/eFsB2Zas/G0pwZPapR292Bz5YeZ83xNDej1DTtFacDSS7z2nt9sGpQl3e2x/Fj9E3mFi6BLO4Cl56cHjs9NpbmzOxZjWolCzFoSQAbTl3O4h5rmpbT6UCSyyilKPHNGKxt8zHsLzv+z/IBYxwKEb+gI0SFZqpNWysLZveqTqVi9vRfdIzNgVezuNeapuVkJg0kSqlmSqm/lVKhSqmhaRz3VUpFKqUCjC+/ZMd+VEqdVkoFKaUmKeNGGkqpd5VSJ4zHfjRl/3MqS2dnXL4cTuGwaEaFV2OJDQwoYE3M/HZwO3PDU3bWFvj3rkGFIvnpt/AI285cy+Jea5qWU5kskCilzIGpQHMM2+Z2VkqltX3uEhHxMr5mGuu+CdQBPIGKQHWggVLKAfgJ8BGRCkBhpZSPqa4hJ8vfujV2jRpRfkUA3xT7iL02lvjaxXN1wdvPlJcrRZs2lszrXZOyLvn4YMERdoVEZnGvNU3LiUx5R1IDCBWRMBF5CCwGUu8FmTYBbAArwBqwBK4C7kCwiDz6G+wvoEOW9jqXUErhMmokysYGrxm7mNJwMuF57OhifZe/f+sAD+9lql17W0vm966Ju2Ne/OYeZu/ZqCzuuaZpOY0pA0lRIPkDCBHGzx7XwThUtVwpVRxARPYB24DLxtdGEQkCQoE3lFKllFIWwNtAcRNeQ45m6eyMy/AvuB8QQLm/zjK3xQJUnoL0MLvGziUdICEuU+0WzGvFQr+alChkSx//wxw8l7k7HE3TcgdTBpK0Ngd/PG/HGqCUiHhiuLuYC6CUeg0oBxTDEHwaKaXqi8hN4CNgCbALOA/Ep3lypd5XSh1WSh2OjHx1h2AeDXFFTpxIqWgrFr29kpK2LnySEM6SZR0gMTFT7TrYWbOwb01cC9jQa85BjvxzM4t7rmlaTmHKQBJByruFYkCKmV4RuS4ij7ZZmwFUNf7cDtgvIndF5C7wJ1DLWGeNiNQUkdrA30BIWicXkekiUk1Eqjk5OWXZReU0yYe4Ln/xBU7WDvi3X0Nd22J8++Ac41a0IzExc9vuOuez4be+tXDKZ43v7IOciIjO4t5rmpYTmDKQHALKKKXclFJWwHvA6uQFlFKuyd62AYKMP1/AMLluoZSyBBo8OqaUcjb+tyDQD5hpwmvIFZIPcd2YOw9bS1smdljDe3lKMvdeGJ+uaMP9+PuZartwfhsW9a1FgbyWdJt5gFMXb2Vx7zVNy+5MFkhEJB7oD2zEEASWishppdRopVQbY7EBxmW8x4EBgK/x8+XAWeAkcBw4LiJrjMcmKqUCgT3AWBEJNtU15CbJh7gehJ3DwtySLzr8wX+tSrA15h/6/N6GqPuZmzgvUiAPi/xqYWdtQbdZBwi6fDuLe69pWnb21P1IlFKlgQgReaCUaohhSe48Eckx4xi5cT+SzIi7do2w1m2wdnOj5MIFKHNziH/Ilt9aMzQ+AgcbB6Y2n0PpAqUz1f4/12N493/7iUtIZPH7tShTOF8WX4GmaS9SVu5HsgJIME6AzwLcgEXP2T/tJXh8iAsACyt83v0d/0RHYu9H0X1tZzad30SiPPskfEmHvCzqWxMzM0XnGQc4G6m37dW0V0FGAkmicZiqHTBBRP4DuD6ljpZNPT7EBYBVXip0Xsmi2Ly4xN7hsx2f0XZVW34P+Z2HCQ+fqX13Jzt+61sTEaHLjP2cj8r4PvKapuVMGQkkcUqpzkBPYK3xM8snlNeyscdXcUmCccWWbSGKdF3F0jsW/BB5nTwx1xm5dyRNVzRl1slZ3H6Y8XmP15zzsbBvTR7GJ9Jlxn7Cb2Tu4UdN03KGjASSXkBtYIyInFNKuQELTNstzZTSHOICsC+KxQc7aFGhO0vOnmF61F3KYM2EoxNosrwJ4w6N40rMlQyd4w2X/Czwq0nMwwQ6z9jPpejMrQrTNC37e+pke4rChiW3xUXkhOm6lPX0ZHtqIkLEx/2J2bMHt5UrsXZ3S1kgMhg2fQkhGwlyLMmckh5sunkahaKFewt8K/hSpmCZp57nREQ0XWccoJCdFUver42LvY2JrkjTtKyW0cn2jKza2o7hGQ8LIACIBHaIyKdZ0M8XQgeStKW5iutxZ7fCxuFwLZCLJWsyv6QHv1/exf34+9QvVh/fCr5UK1wNY3LmNB29cJPuMw9QOL8Niz+ohXM+HUw0LSfIylVb9iJyG2gPzBGRqsBbz9tB7eVLd4grudKN4INd0HI8RSNDGbpzFpvyVOLjcj04GXmS3ht703V9Vzb/s5mEdJ6Qr1KiIP69a3DldixdZxwg6u6DNMtpmpYzZSSQWBifQO/Ev5PtWi6R5iqux5lbQPU+MOAYvPkJBU6v4sPNP7OpcDO+rPZfoh9E8+n2T2mzqg1L/15KbHxsqiaqlyrErJ7VCb95j24zD3Az5tlWg2maln1lZGjrHeArYI+IfKSUcgd+EpEck75dD209WYaGuJK7EQabR0LQashfjASfEWyxL8DsU3M4ff00hWwK0bVcV94t+y721vYpqu4OiaL33EOUcbZjkV8t7G31AkBNy66ybI4kN9CB5OlurV7Npf8Owfm//8Whd6+MVTq/GzYMgysnoGg1pOl3HLYyY/ap2ey+uJs8FnnoUKYD3ct3p4hdkaRq2/++xvvzjlDONR/z/WqS30YHE03LjrJysr0YMBnDjoUC7AYGikhEVnT0RdCB5OmeuoorPYmJcPw32PI13L0KFTvAW6MIlljmnp7L+rD1CEIzt2b0qtCLsoXKAvBX4FU+XHAEz2L2zOtTEztrC9NdnKZpmZKVgWQzhpQo840fdQO6ikjj5+7lC6IDScY8GuJSVpa4DB9OvqZNn7gaK4UHd2HPBNg72fC+dn+o+x+uxN9lfuB8lgcv5178Pd4s8ia9KvaipktNNp6+wseLjlG1REH8e1fH1koHE03LTrIykASIiNfTPsvOdCDJuNigIC4P/5LYwEDsGjbEZeQILF2fISNOdLjh7uTkMrArDD4joFIXbsXdYVnwMhYELuB67HXKFSpH74q9ib1VgU+XnKCmmwOzfauTx+op8zOapr0wWRlI/gL8gd+MH3UGeomIz/N28kXRgeTZSHw8N+YvIHLSJJRSOA0aSMGuXZ8+CZ9c+CHYOAwiDoGLJzT7HkrV5UHCA9aeXYv/aX/O3z5PUbuiVM7flsXbXKhbuggzelTDxlIHE03LDrIykJQApmBIkyLAXmCAiFzIio6+CDqQZM7DiItc+fprYnbtwsbDA9dvRmPzxhsZb0AETq0wrPC6HQFvtIIm30AhdxIlkW3h25hzag7HI4+Txzw/0VeqU8OhNbO6N8TaQgcTTXvZTLpqSyk1SEQmZKpnL4EOJJknItxev56r331PQnQ0Dr174divH2Z58mS8kbj7sG8K7PoZEh5CrQ+h/mCwMSwNPnbtGLNPzWZ7+HYk0RIXs3pMb/MZ7gVLmOiqNE3LCFMHkgsikmP+lOtA8vwSoqO5Om4ct5avwLJ4cVxGjcSuTp1na+TOFdjyDQQsBNtC4P0FVPE1PPAIhEWH8eW2KZy4tRWlEmlSsjG9PXpTwaFC1l+QpmlPlZUpUtJsP4OdaKaU+lspFaqUGprGcV+lVKRSKsD48kt27EfjNrxBSqlJyrh8SCnVWSl1Uil1Qim1QSnlmMlr0J6BeYECFPn2W0rMnYsyNye8jx+Xhgwh/saNjDeSzwXengrvbwencrDuM5hWF0L/AsC9gDuL2o3Hr+QMHkTVZ8s/u3hv7Xv4bfRjz8U9vArPPGlaTmSyOxKllDkQDDQGIoBDQGcRCUxWxheoJiL9H6v7JvATUN/40W5gmPG/l4DyIhKllPoRuCcio57UF31HkrUSHzzg+v/+R9SMmZjnzYvz0CHYt22b8aXCYJg/ObMWNn0FN89BmSbQ5FtwMjxnMm3HWcZuDKBKhTNEW24l8v41Xi/4Or4VfGnm1gxLM/0Qo6aZ2nPfkSil7iilbqfxugMUSa9eMjWAUBEJE5GHwGKgbQb7L4ANYAVYY9hI6yqGOyEF5DXeoeTHEFi0F8jM2hqnAQNw/30FVm5uXB46jAu9e/Pwn38y3ohSUK41fHwAGn8DF/bDL7Vh/WC4d4MPG5TmMx9Pjp70oqr6idFvfkNCYgJf7P6CFr+3YH7gfO7F6Q2zNC07SDeQiEg+EcmfxiufiGTkybGiQHiy9xHGzx7XwThMtVwpVdx47n3ANuCy8bVRRIJEJA74CDiJ8c4Ewz7y2ktgXaYMJRcuwGXkCGJPniKsTVuips9A4uIy3oiFNdQZYEgIWdUXDs2ESV6wbyqfNCjJAJ8yLDtymcOnSrOizQqmNJpCUbui/HjoRxovb8yko5OIuh9lsmvUNO3pMjtHkhFpjXM8Po62BiglIp7AX8BcAKXUa0A5oBiG4NNIKVVfKWWJIZBUxnBXdALDkFfqkyv1vlLqsFLqcGRkZFZcj5YGZWZGwc6dcV+3Drv69YkcP55zHTpy//jxZ2soryO0Gg8f7oGiVWHjF/BLLf5TPISPGriz8MAFvll7hvrF6uPfzJ8FLRZQw6UGM0/OpOnypny972vO3zpvkmvUNO3JTJa0USlVGxglIk2N74cBiMj36ZQ3B26IiL1SajBgIyLfGI+NAGIx3KWMffQwpFKqPjBURFo8qS96juTFubNlC1dGf0P8tWsU7NoVp0GDMLfL+2yNiEDIZtg0HKKCEbf6zMjTl++OmuNX143hLcslzcecv3WeeYHz+CP0D+IS4/Ap4UOvir3wdPI0wdVp2qvF1Ku2MuIQUEYp5aaUsgLeA1YnL2Dc5+SRNkCQ8ecLQAOllIXxLqSB8dhFoLxSyslYrnGyOlo2kM/HB/d1aynYpQs3Fy4krFUr7mzd+myNKAWvN4GP9kLzn1BXTtI3qCfLiy5m1e4Aftz4d9IKrlL2pRhRewQbO27Ez8OPA1cO0HV9V3w3+LIjfAeJkmiCq9Q0LTmTppFXSrUAJgDmwGwRGaOUGg0cFpHVSqnvMQSQeOAG8JGInDHenfyCYdWWABsebe2rlPoQGAjEAf8AviJy/Un90HckL8f9gAAufzWCByEh5GvShMLDh2NZ2DkTDd2EHT8iB6fzACsmPmhDnvr9GdDUI1XRe3H3+D3kd+YFzuNyzGVK25fGt6IvLd1aYmmuV3pp2rPIyhQpd0g9t3ELOAx8JiJhme7lC6IDycsjcXFcnz2HqKlTUVZWOH/+GQU6dUKZZeJmOCoU2fQlKvhPwhOdOFXhM5p3+tBwB/OYuMQ4Np7fyJxTcwi+GYyzrTPdy3Wn4+sdsbOyy4Ir07TcLysDydcYVkgtwjCB/h7gAvyN4Q6i4XP31sR0IHn5Hp4/z+VRX3Nv/37yVK6M6+ivsS5TJlNtJYRu5+qyTyny4CyX7b1w7fQzFK2SZlkRYe+lvcw5NYcDVw5gZ2lHp7Kd6FquK862mbg70rRXSFYGkgMiUvOxz/aLSC2l1HERqfScfTU5HUiyBxHh1qo/uDZ2LAn37uHY1w+HDz7AzNr6mdtKiI9n6YzveevKdJzUbfB8z5Cy3j6tFeYGp6+fxv+UP5v+2YSZMqO1e2t8K/jiXsD9eS5L03KtrAwk+4CfgeXGjzoCnxoDSY7Yl0QHkuwl/sYNro4dy+3Va7AqVQqX0V+Tt0aNZ24nLiGR/y7cTZngGXxgtQFzM3OoOwje/ASs0l8pFn4nnHmn57EqdBWxCbE0LN6Q/1T9D+72OqBoWnJZGUjcgYkY0sgD7AP+g2EFVVUR2f2cfTU5HUiyp7u793Bl1CjiIiKw79iBwp9/jnmBAs/UxsP4RPotPMqZMydZVHI9Ja5sgnxF4K2R4NEJnjAXczP2JovPLGZB0ALMlTnTm0znjULPkCZf03I5k2b/zWl0IMm+Eu/fJ2rqVK7P8ce8QAEKfzGM/C1aPFPergfxCXww/wg7giOZ5R1Po/M/w6VjUKSKYUOtErWeWP/C7Qv02dSHe3H3mN54OhUcdbZhTYMsfI5EKVVMKbVSKXVNKXVVKbVCKVUsa7qpverM8uTB+fPPcVu+DEtXVy599jnhH3zAw4iLGW7D2sKcad2qUvc1R/pss2BVtfnQ7n+GtPWzm8IyX7iZfh6wEvlL4N/Mn3xW+fDb5EfAtYAsuDJNe3VkZA3mHAwPEhbBkK5kjfEzTcsyNuXKUWrJYgp/MYx7h48Q1ro11+f4I/HxGatvac707tWo5ebAp8tOsE41gE8OQ4Oh8PcGmFId/hoFsbfTrF/Urij+zfxxyOPAB5s/4PAVfQeraRmVkUDiJCJzRCTe+PIHnJ5WSdOelTI3p1CPHpReu4a8NWpw7YcfON/pXe6fPp2h+nmszJnlW41qJQsxYPExNgTfAe9h8MkRqNAOdv8Mk6vCkbmQmJCqvkteF+Y0nUPhvIXpt6Uf+y/vz+pL1LRcKSOBJEop1U0pZW58dQOe+CS5pj0PyyJFKDbtV4pO+Jm4a9c4/04nrv7wI4n3np423tbKgtm9qlOpmD2f/HaULUFXDUuC2/8P/LZCITdYMwD+1wDCdqSq72TrxOymsylqV5T+W/qz5+IeU1yipuUqGQkkvYFOwBUMKd07Ar1M2SlNU0qRv1kzSq9bS4GOHbkxZw5hrVpzd+fOp9a1s7bAv3cNyrnm56MFR9n+9zXDgWJVofdG6DgHYm/BvDbwW2e4fjZFfcc8jsxuOhs3ezc+2foJ28O3m+AKNS33eGogEZELItJGRJxExFlE3gbav4C+aRrm9va4jv6akgvmo2xsCH//Ay5+9jnxUU/egyS/jSXze9fkNWc73p9/hN0hxvJKQcX20P8Q+IyEczthag3YPcGQddiooE1BZjaZSdmCZfnPtv+w+Z/NprxMTcvRMpv999Ms7YWmPYVttWq4rVqJY//+3Nm0ibMtWxG9YsUT93G3t7VkgV9N3B3z4jfvEPvDko3IWtpAvU/hk6PwRiv4aySsHQQJ/07u21vbM73JdCo6VmTwjsGsD1tvykvUtBwrs4HkGTbn1rSsYWZlhVP/j3FbtRLrMq9xefiXXOjpy4Nz59KtUyivFQv8alKsoC29/Q9x+PyNlAXyFTYMddX7DI74w2/vwoM7/x62yse0xtPwcvZi2O5h/BH6h4muTtNyrswGktz/FKOWbVmXLk3JefNwGf01sUFBnGv7NlG//oo8fJhmeUc7axb51cQlvw2+cw5x7MLNlAXMzAx5ulpNgLPbYE5zuH056XBey7z8+tav1HCpwVd7vmJF8ApTXp6m5TjpBhKl1B2l1O00XncwPFOiaS+NMjOjYKdOuK9bi51PIyInTiKsfXvuHT2WZnnn/DYs6lsLBzsresw+yMmIW6kLVesFXZbAjXMw8y24Gph0KI9FHqb4TKFO0TqM2jeK3878ZqpL07QcJ91AIiL5RCR/Gq98ImLxIjupaemxdHam2M8/U+zXX0iMucc/Xbty+euvSbhzJ1VZF3tDMLHPY0m3WQcIvJTGw4llGkOvPyEx3vBUfNj2pEPW5tZM9J6Id3FvvjvwHfNOzzPhlWlazmHKrXY17YXJ5+1N6bVrKNSjO9FLlhLWoiW3N21KVa5ogTz81rcWea3M6TbrAH9fSR1wcPUEv7/Avhgs6AABi5IOWZlb8X8N/4/GJRvz0+GfmHlypikvS9NyBJMGEqVUM6XU30qpUKXU0DSO+yqlIpVSAcaXX7JjPyqlTiulgpRSk5RBvmRlA5RSUUqpCaa8Bi3nMMubl8LDhlFqyWLMHRy4OGAg4R/3J+7KlRTliheyZVHfWliaK7rO3E/otTSCSYHi0HsDlKoLqz6C7WOTlgdbmlnyY/0faeHWgolHJ/Lr8V+fuHpM03I7kwUS477rU4HmQHmgs1KqfBpFl4iIl/E101j3TaAO4AlUBKoDDUTkTrKyXhj2bP/dVNeg5Ux5PDxwW7YU58GfE7NnD2EtW3FjwUIk4d+0KKUc87Koby1A0XnGAc5cSWOYy8YeuiyDSl1g+/fwx8cQb5jQtzCz4Lu639G2dFt+CfiFyccm62CivbJMeUdSAwgVkTAReQgsBtpmsK4ANoAVYA1YAleTF1BKlQGcgV1Z1mMt11CWljj06YP7mtXkqVSJq99+yz9duhL7d3BSmdJOdvzWtyYiQqtJu/n+zyBiHjyWJNLCCt7+BRoOg4CFsLCj4al4wNzMnNF1RtPx9Y7MODmDcYfH6WCivZJMGUiKAuHJ3kcYP3tcB6XUCaXUcqVUcQAR2Qdsw5CS5TKwUUSCHqvXGcPdjP6Tq6XLqnhxis+aSZGffuThhQuc69CBaz9PIDE2FoAyhfOxYVB92lUuyv+3d+fxUVXnH8c/z2SSTPaFhH2PYFFAKqAiFnBHFKjgAqgVLUSt1qrV1qo/61JL61KXigsIiiibIAgqiFhBRVBxYxcI+5qQhITsmZnn98dccIhRAmEYAs/79ZoXc+89d+acCbnfnDP3nvvygvWc/9QC3lu648BAEIFe98JvX4RNC2FsbyjYCoBLXDx41oMM+dUQXl/5OiO+HIFf/eFoqjFhE8ogqe6ixaoH/VlAS1XtCMwDxgGIyElAO6ApgfA5T0R6VNl3EPCz52CKSKaILBGRJTk5OYfZBHM8EBV5biwAACAASURBVBGS+val9fvvkdS3L7kvv8z6/v0pXhyY3TctPponrjyNabd0IzUuilsnfMN1Y75kXXbRgS/UaQhcOy0QIqPPhx3f73/9e8+4l6GnDmXi6ok8sugRCxNzQgllkGwFmgUtNwW2BxdQ1VxVLXcWRwOdneeXA4tVtUhVi4DZwP7b3InIaYBbVb/+uTdX1VGq2kVVu6Sn26z3BtwpKTQe8U+avzoWgM1Db2D73+7Dmx+4QLFzi1Rm3tadh/udyvdb93DJs5/w7zmrKakIGu5q3Ssw8aPLDa/2gbWBObhEhLs638XwDsOZtnYa/7fw//BVM1W9McejUAbJV0AbEWklIlEEehAzgwuISKOgxX7AvuGrzUBPEXGLSCTQM2gbBIa17Iowc1jiunWj9TvvUO+mmyiYNYv1fS5lz9vTUb8fd4SL689uyf/+3Iv+nZrw4vwsLnhqAe8vCxruanBK4PTg1FYw4WpYErjPm4hw++m3c2unW5mZNZP7PrsPr79mN+Yypi4LWZCoqhe4DfiAQAhMUdUVIvKIiPRzit3unOL7PXA7MNRZPxXIApYB3wPfq+qsoJe/CgsSUwsuj4f6d95Bq2nTiGrRgh333cfGQYMpXboUgPSEaJ688jSm3tyNxJhI/vDmN/xu7Jesz3GGuxIbBS5czDgvMNnjvIfBHxjOuvm0m7nj9Dt4f8P7/OWTv1DprwxXM405KuRE+K66S5cuumSJ3TrVVE/9fgpmziT7qafw5ewmacAA6t95B25nSNTr8zN+8Sb+M3cNZV4fmT1ac+u5JxEb5Q7MFvy+M+Fj+ysCZ3i5owF4fcXrPLHkCXo168VTPZ8iKiIqjK005tCJyNeq2uWg5SxIjAnwFRWT+9KL5I57HVdUFGm33krqtdcgUYEAyN5bxr9mr+btb7bROMnDg31P4eJTGwbOKvnsafjoYWjRHa5+A2JTAZi0ehKPffEY3Zt055lez+Bxe8LXQGMOkQVJEAsScyjKN2wg+1//pmjBAqJataLBfX8j/je/2b/9yw15PPjOclbv3EuPtuk83O9UWqXFwbKpgavgU1rCNW8F/gWmrZnGw4se5oxGZ/Dcuc8RGxkbnoYZc4gsSIJYkJjDsXf+fLJH/IuKTZuIP/dcGtz7V6JatAACw12vL9rEfz5cQ4XXv3+4K2b7Ypg0BCIiAzMJNwmciDgraxYPLHyAX9f/NSPPH0lcZFw4m2ZMjViQBLEgMYdLKyrIGz+e3SNfQCsrSR06lLSbb8IVFwiC7MIyRsxezfRvt9EkOYYH+57CRfULkDevgKIcuGIs/KoPAHM2zOHeT++lfVp7XrzgRRKiEsLZNGMOqqZBYrP/GvMLJCoqMNXKnNkkXnopuaNHk3VJHwpmzkRVqZ/o4emrOzE58yzio93cNP5rbni3gM0DZkH9djD5GvhiFAC9W/XmyZ5PsiJ3BZlzMykor+aeKMbUQdYjMeYQlH73HTv/8Rhly5cT06kTDe6/n5gO7QGo9PkZ9/lGnpm3lgqvn9vOacStef8iYu1s6HYbXPgouFws2LKAO+ffSUZyBqMuHEWKJyXMrTKmeja0FcSCxBxJ6vdTMH0G2f/5D768PJIGDqD+nXfirlcPgF2FZfzz/VW88912midH8UbTd2i+bjy06wcDRkFkDAu3LeRPH/+JZgnNGH3RaNJi0sLcKmN+yoa2jAkRcblIHjiAjDmzSb3+egpmvENW70vIGzcOraykQaKHZwf9monDzyI6Kooey3szMeVmdNUsGNcPinfTvUl3Rp4/km1F27jxgxvJLskOd7OMOWzWIzGmlsrXr2fXP0dQ/NlnRGVkBE4X7t4dCAx3vbZwI8/MW8O5uphnIl/AldQE17VToV4GX+/6mj/M+wNpMWmMuXgMDeMahrk1xvzIeiTGHCXRrVvTbPQomr7wAlpRwZbfD2PLbbdRsWULkREuhvdozUd/7oWc0p+rSv9GYf5uKl4+DzZ/QecGnRl10Sjyy/IZOmcoW/duDXdzjDlk1iMx5gjyl5eT99o4dr/8Mni9pN54A2mZmbhiAxchfp61m5emf8jDhX+naUQee3q/QPqZV+0/kys2MpYxF42heWLzMLfEGOuRGBMWruho0m7KJGP2+yRcfDG5L70cOF343fdQVc7OSOOVO67mkx4TWOZvRb33M/nktQfJSDiZsRePpdxbztA5Q1lfsD7cTTGmxqxHYkwIlXzzDbv+8RhlK1cS07kzDe+/D88ppwCwIzePHa8O5fSiBUxz9yFlwFM0b1TCsLnDUJRXLnqFNiltwtwCcyKzHokxx4DY00+n5VtTaPjIw1SsX8+GgVew4+8P4c3Pp1G9VE6/awbb2g1joPd9fBOv5T8zdjCi24u4xc2NH9zI6rzV4W6CMQdlPRJjjhJfQQE5I0eS/+YEXHFxpP/xj6QMHoS43XgXj8I156+s0Jbc7PsLvX/ThAWFj1LqLeHlC1+mfVr7cFffnIDsgsQgFiTmWFK+di07//lPShYtJrpNGxrcfx9xZ50FP8zG/9YN5JHE1cV/pjwtFXeTUZT79/LiBS/SqX6ncFfdnGBsaMuYY1R0mzY0HzuWJv99Dn9JCZuH3sDW2/9ERVxHXDe8T1q0nzkJj9LFu5NNy6/HWxnH8LmZLNlpfwyZY1NIg0REeovIDyKyTkTurWb7UBHJEZHvnMewoG2PO7fhXSUiz4mIOOujRGSUiKwRkdUiMjCUbTAmFESExAsvpPV775L+p9sp+vRT1l96KTnTFuK/9l0iExvxdMXDjOqQR9GGTEpLExg+9yY+3fp5uKtuzE+ELEhEJAIYCVwCnAIMFpFTqik6WVU7OY9XnH3PBroDHYH2QFegp1P+fiBbVds6r7sgVG0wJtRcHg9pt9xCxvvvkXD++ex+4QWyhtxCYfO/QpOuXLjqARafvZ4zo++nvDSVP8y7jRe+eDfc1TbmAKHskZwBrFPV9apaAUwC+tdwXwU8QBQQDUQCu5xtNwIjAFTVr6q7j2itjQmDyEaNaPKfp2gx/nUikpLY9pcH2Py/VMrS+5D0+QjGJM/gibOfJ8LbgBdWPsBVr7/Ctj2l4a62MUBog6QJsCVoeauzrqqBIrJURKaKSDMAVV0EfAzscB4fqOoqEUl29nlURL4RkbdEpEEI22DMURXbtSutpk2l4UN/p3xdFhtGLmXn9p54F73Opd/fx4cDR9HA05qV/v9ywUv/ZeTH6yj3+sJdbXOCC2WQSDXrqp4iNgtoqaodgXnAOAAROQloBzQlED7niUgPwO2sW6iqpwOLgCerfXORTBFZIiJLcnJyjkR7jDkqJCKClEGDyJgzm5TBg8n/LIv1H7Yif85i0iYO4p2L/s2pqe1xN3qTpxdNpvczn/LJGvs/bsInlEGyFWgWtNwU2B5cQFVzVbXcWRwNdHaeXw4sVtUiVS0CZgNnAblACTDdKfcWcHp1b66qo1S1i6p2SU9PPxLtMeaoikhOpuH/PUCr6dOJPvU0di5JZMP4HGREP17tfBddGpxObJPJlEYv5ndjv+SWN75muw13mTAIZZB8BbQRkVYiEgUMAmYGFxCRRkGL/YBVzvPNQE8RcYtIJIEv2ldp4KKXWUAvp9z5wMrQNcGY8POc3Jbmr71Kk2eewRfVgM3vQv7vr+a5hIs5q9FZlCRO4NKzN/HxD9mc/9QCXpi/jgqvP9zVNieQkF6QKCJ9gGeACGCsqj4mIo8AS1R1poiMIBAgXiAPuEVVVztnfL0A9CAwHDZHVe9yXrMFMB5IBnKAG1R18y/Vwy5INMcLf2kpuS88Q+7Y1wEl+fIePHa2MD/nc25pfzffrTiVuSt30To9jkf6teecNnbnRXP47Mr2IBYk5nhTuX41u+66jr2ri3DXi+PdARmMrbeCu7veQ/OIS3ho1go25ZZwaYdGPHBZOxolxYS7yqYOsivbjTmORbb+FU2nfU7z4Z2I8O6h9+ilPD09mUmznyCrciYf3NGDuy5sy7xVuzj/qQW8tCDLhrtMyFiPxJi6TBX9+N/kj3mWnBWpeCuUOadD4s3DGdb9Drbml/LwrJXMW7WLjPQ4Hunfnu4n2XCXqRkb2gpiQWKOe99NxDvlj2SvbUz+8kqKPLBlSA8G3DkSl9vNR6t28dCsFWzJK+Wyjo144NJTaJjkCXetzTHOgiSIBYk5IaxfAJOvo6QwhiVLU0nP2kNBi3p0eOwZ4rp0oazSx0sLsnhhfhZul3BGq1Qy0uM5qX48GenxZKTHUS8+OtytqBWtrMRfXIyvqBh/cRH+osDDV1QEXi9x55yDu169cFezzrAgCWJBYk4Y2avgzSvxF+fxRkQ3Wr29krS9kHBpHxrccw+RDRuyObeE//5vLSu2F7J+dxFllT9+d5ISG3lAuOz7t0lKDBGu6q4xrj1VRUtLnQAowh8UAvuXi4oC64LL7AuJ4h+Xtbz8F99LIiNJ7HMJKUOG4OnYEWcuWPMzLEiCWJCYE8renfDmleiuFTzX/jL2vreE334puCOjSbvpJlJvGIorOtDz8PuVbXtKWZdTRFZ2EVk5RWRlF5OVU0RuccX+l4x2u2iVFkdG/XhOSo8no348GanRtIwRospL8RUX/3wA7DvY7ytTTQDgr8GJAJGRRMTH44qLwxUfjys+joi4eOe5s7xve1yVdfHx+MvKKHh7OgUzZuAvLsbTvj0pQ4aQ2OcSXB4b5quOBUkQCxJzwikvgqk3oGvn8nzHi5i+ZRV/XdyA5t9uJ7JZM+r/5R48bdo4Q0DVB0DpngIKcgsoyS+gvDCwTUqKiSovJdZbhsdXWaOquGJjAwf14ACIj6/2YP9jCASt2/eIijoiH42vqJiCme+QP2ECFeuyiEhKIvnKK0geNJioptVNB3jisiAJYkFiTkg+L7x/N3z9Ki+17cbIym3cWNqFfrNyqMjK+uV9IyIOPLgHHfA1Jo69EVHkEUm2z832ygi2lMHGUtgjUZS4PZS4PUQlxtOkUT0yGiYetWGyQ6GqlHzxJfkTJrD3o4/A7ye+Vy9ShgwhrvvZiMuujrAgCWJBYk5YqrDwWZj3d8a26MDTrgIuano+D5RfiMvrO2AYKCI+bv9f/xIdfcjfH/x0mKx4/3DZQYfJ0uPISI/HExlxKG8IlcVQUQIVRVBRDJVBz/etrywJLHvLIbExpLQMPJKbQ2TgQs3KnTvJnzyZPVPewpebS1SLFqQMGUzS5ZcTkZh4SJ/D8cSCJIgFiTnhLZsKM25hfP0mPO7x0atpLx7v+Tgx7qNwxbvfR35BAZt3ZLM1ezc7cnaTk5vH7vw9lBQVEKtlxEg58VJGfY+PxjE+0qO91IvykhRRQZyUEeUvc8Ih6OE9xAkqXZHgrzIcl9DICZUWkNISf3xT9i7fTf7shZQuXYHExJDUty8p1wzBc/LJR+wjqSssSIJYkBgDbPocJg5mcryHfyREUc9Tj+tPvZ6rTr6KuMg48Pt+/Ou96qOyunW/8Nd/RZGz7tAP+KV4KNZoijWaEqIpwUOFKwZXdByRMQl44hKJj08iKSmZhMQkXNFxEBUPkbEQ5TyPcp5Hxjn/xoIIFO+G/I3VPwq3EXyni7LCOPI2plG4xot6lZi2jUntdz4Jl1yGpGcEXvc4Z0ESxILEGMfutfDGQL6pyOOltDQWRfhI8ivX7i1hyJ48Ev2HcDyIjPvxgH3AgTzoERnrHNj3lY2vsr7K/pGx4HLtHybLyili3dEYJoPA0NeeLU6wbNgfML4dG9izZAf5q91UFrtxe3wkZ5SQ3MFDZNOWPw6VBT8SGoHrEN//GGRBEsSCxJggRTnw4YNQXsiyCD+jKncyv3I3ceJmcHJ7rkvvSmps/Z8PgqADfjjkF1cETlMOCpl12UVsyS9h3+FMBJokxxxwPUzrtDgaJ8dQPzGaaPchHuRV0eJciue+Q95bMyn+dg24hIST40htW0pMzDaEoFOYI6IC38Hs/z6mxYFB46kb37tYkASxIDHml/2Q9wOjl41m7sa5REdEc+XJVzL01KHUj60f7qrVWFmlj425gVDJyi7e/6V/1YsuAVLjomiQ6KFBYjQNEz00SPTQMCmw3CDRQ8NED6lxUT97wkHFpk3kT5zEnrffxl9YSHTbNqT89mKSzmyNq3SH05vZ9OOwWdmeA18gJrX6nkxKS0hsAhHuI/zpHB4LkiAWJMbUzPqC9YxZNob31r+HS1xcftLl3NjhRprE193rK/YNk63fXcyugjJ2FgYeuwrK2LW3jJ0F5eQWl1P1UBgV4SI9IZqGSZ79YdMgMdoJnMC69EilYu5s8idMpHzVKlwJCSQPuJyUwYOJatnyxxcrzT8wWIIfBVvA7/2xrMsNSc0gpUX1QROTEsJP60AWJEEsSIw5NFv3bmXs8rHMWDcDv/q5rPVlDOswjJZJLcNdtZCo9PnJ3lvOLidg9oVNdmE5OwvK2OUsl1T4frJvosdNw8RoOu/dSvflH9Nq5Ze4fF7KOnUlcuBVpF9wLmlJsT9/7YzPG/iiP38j7KkmbEpyDyzvSao+YJJbBALIfWQu3AQLkgNYkBhzeHYW72TcinFMXTOVcl85F7e8mGEdhnFy6ol3KqyqUlTuDYRKQXmgV+M89oXNrsJyKnOyuWjDF/TZuIi0skJ2xqYwu/XZfHPqb4ivn/bjcNoBPZ3A0Fp8dDVDWmWF1QdM/kbYsxl8P558gLggsemBvZnTfwfxhzdEeUwEiYj0Bp4lcKvdV1T1X1W2DwWeALY5q55X1VecbY8DlxK4+daHwJ9UVUVkPtAI2HdO4UWqmv1L9bAgMaZ2cktzGb9yPJN+mERxZTHnNjuXzI6ZtE9rH+6qHXN8fmV3UTk7c4so/OgjomdNI+GHZXjdkaz41ZnMbfMbvopuwN4y70/2jY92U98Jmn1h0yAhaDgtyUN6fDTuCOdEB78f9u74+VOai7Ph9u8gtdVhtSXsQeLcd30NcCGwFfgKGKyqK4PKDAW6qOptVfY9m0DA9HBWfQb8TVXnO0Fyt6rWOBksSIw5MgrKC5iwegJvrHyDwopCzm58NpkdM+ncoHO4q3ZMK1uzhvyJEyl4ZyZaUoLntI7EXTWIom692FXq3z90dmAPJzDU5q1ySrYIpMVHH/i9TZUeTsNED4kxbqSyBNwxh32G3bEQJN2Ah1T1Ymf5bwCqOiKozFCqD5JuwPPAOYAAnwDXqeoqCxJjwq+4spjJP0xm3Ipx5JXl0blBZzI7ZNKtcTebmv0X+PbupWCGM2Hkhg1EpKaSfMUVpAy6msjGjX9S3u9X8koqDvieZldh+f7vcfYFT37JTyfQ9ES6aJjoYczQrmSkxx9WfY+FILkC6K2qw5zl64Azg0PDCZIRQA6B3sudqrrF2fYkMIxAkDyvqvc76+cD9QAfMA34hx6kERYkxoRGqbeUt9e+zdjlY8kuyaZ9vfZkdsykV7NeFii/QFUpWbSIvDcnUPTxxwDEn3cuqddcQ+xZZx3yZ1dW6QucGFClV7OzsIyH+5162DcsOxaC5Erg4ipBcoaq/jGoTD2gSFXLReRm4CpVPU9ETiLw3crVTtEPgb+q6ici0kRVt4lIAoEgeUNVX6/m/TOBTIDmzZt33rRpU0jaaYyBCl8FM7Nm8sqyV9hWtI22KW0Z3mE4F7a4kIjj4ArvUKrcto38yVPY89Zb+PLziWrdmpQhQ0j6bX8i4g+vJ3GkHAtBctChrSrlI4A8VU0SkXsAj6o+6mx7EChT1cer7DOUaobGqrIeiTFHh9fvZfaG2YxeNpoNBRtomdiSYR2G0ad1HyJdkeGu3jHNX17O3jlzyHtzAmVLl+KKjSXpt/1JGTKE6JNOCkudahokoZzj4CugjYi0EpEoYBAwM7iAiDQKWuwHrHKebwZ6iohbRCKBnsAqZznN2TcSuAxYHsI2GGMOgdvlpm9GX6b3m85TPZ8iOiKaBxY+QN/pfZnywxQqgk9VNQdwRUeT1L8/raZMpuVbU0i46CL2TJ3G+sv6sun6oRR+MBf1/vRMr2NBqE//7QM8Q+D037Gq+piIPAIsUdWZIjKCQIB4gTzgFlVd7fROXiBw1pYCc1T1LhGJI/DFe6TzmvOAu1T1p1cJBbEeiTHhoap8svUTRi0dxdLdS6kfU5+h7YcysM1AYiNjw129Y543P589U6eyZ+IkKrdvx92gASmDrib5yitxp6WF/P3DPrR1LLEgMSa8VJUvdn7BqKWj+GrnV6R6UrnulOsYdPIg4qPC+z1AXaA+H0ULFpD/5gSKFy6EyEgSL76YlGuGENOpU8hObLAgCWJBYsyx49vsbxm1dBSfbfuMhKgErml3Ddf86hqSPcnhrlqdUL5hQ+CalLen4y8qIvqUdqQOGULipZfiijmyNyqzIAliQWLMsWdF7gpGLx3NR5s/ItYdy9UnX83vTv0daTGhH7I5HviLiymY9S75EyZQvmYNrqQkkgcMIGXwIKKaNz8i72FBEsSCxJhj19r8tbyy7BXmbJxDpCuSgW0GckP7G2gY1zDcVasTVJXSJUvImzCBvR/OA5+PuB6/IfWaa4g75xykFveNsSAJYkFizLFvU+Emxiwbw6ysWSDQP6M/v2//e5olNgt31eqMyl3Z7Jkyhfwpk/Hl7CayeXOaPv9fPG3bHtbrWZAEsSAxpu7YXrSdV5e/yttr38arXvq06sOwDsPISM4Id9XqDK2oYO+8eeyZMYOmzz572N+dWJAEsSAxpu7JKclh3IpxTFkzhTJvGRe0uIDhHYbTrl67cFfthGFBEsSCxJi6K78snzdWvcGEVRMoqiyiR9MeDO8wnE71O4W7asc9C5IgFiTG1H2FFYVMWj2J8SvHs6d8D2c2PJPMjpl0bdjVJogMEQuSIBYkxhw/SipLeGvNW7y24jV2l+6mU3onMjtmck6TcyxQjjALkiAWJMYcf8p95UxfO52xy8eyo3gH7VLbkdkxk/Oan4dLQjmN4InDgiSIBYkxx69KXyXvrn+XV5a9wua9m8lIymBYx2H0btkbt6uae6CbGrMgCWJBYszxz+v3MnfjXEYvG826PetoltCMYR2G0bd1XyIjjr0p7FWVCn8F5b5yyr3llPvKqfA5y0GPCl8FZb6y/dsqfBWUect+fluV5WfOfYbG8T+9+2JNWJAEsSAx5sThVz8fb/mYUUtHsTJ3JQ3jGnJj+xu5/KTL8bg9B5QNPpjvP4h7y6s9mFe3XOYNOmj7Kw5Y/rl9g9fVhktcREdEEx0RTVRE1P7nwes8ER7uO/M+GsU3OvgLVsOCJIgFiTEnHlVl4faFjFo6im+zvyUxKpGEqISwHMz3bYuKiMLj9tS8bESVsu4fy7rFHfKTC2oaJDaAaIw5LokI5zQ5h+6Nu7Nk1xJmrJuBqh70YP5LB/dwHMzrAgsSY8xxTUTo2rArXRt2DXdVjlt2jpwxxphaCWmQiEhvEflBRNaJyL3VbB8qIjki8p3zGBa07XERWSEiq0TkOanSfxSRmSJi92s3xpgwC9nQlnPf9ZHAhcBW4CsRmamqK6sUnayqt1XZ92ygO9DRWfUZ0BOY72wfABSFqu7GGGNqLpQ9kjOAdaq6XlUrgElA/xruq4AHiAKigUhgF4CIxAN3Af844jU2xhhzyEIZJE2ALUHLW511VQ0UkaUiMlVEmgGo6iLgY2CH8/hAVVc55R8FngJKfunNRSRTRJaIyJKcnJxaNsUYY8zPCWWQVHdOXNWLVmYBLVW1IzAPGAcgIicB7YCmBMLnPBHpISKdgJNUdfrB3lxVR6lqF1Xtkp6eXpt2GGOM+QWhPP13KxB8j8ymwPbgAqqaG7Q4Gvi38/xyYLGqFgGIyGzgLGAv0FlENhKoe30Rma+qvULRAGOMMQcXyh7JV0AbEWklIlHAIGBmcAERCb5uvx+wb/hqM9BTRNwiEkngi/ZVqvqiqjZW1ZbAOcAaCxFjjAmvkPVIVNUrIrcBHwARwFhVXSEijwBLVHUmcLuI9AO8QB4w1Nl9KnAesIzAcNgcVZ11uHX5+uuvd4vIpsPcPQ3YfbjvXUdZm08M1ubjX23b26ImhU6IubZqQ0SW1GSumeOJtfnEYG0+/h2t9tqV7cYYY2rFgsQYY0ytWJAc3KhwVyAMrM0nBmvz8e+otNe+IzHGGFMr1iMxxhhTKxYkjhrMVHyziCxzZin+TEROCUc9j6SDtTmo3BUioiJSp892qc1s1HVVTX7GInKViKx0ZtuecLTreKTV4Of8dNDPeI2I7AlHPY+kGrS5uYh8LCLfOlNS9TmiFVDVE/5B4DqXLKA1gYkivwdOqVImMeh5PwLXtoS97qFss1MuAfgEWAx0CXe9Q/wzHgo8H+66HuU2twG+BVKc5frhrneo21yl/B8JXOMW9rqH+Oc8CrjFeX4KsPFI1sF6JAEHnalYVQuDFuP46bxhdU1NZ2d+FHgcKDualQuB2sxGXVfVpM3DgZGqmg+gqtlHuY5H2qH+nAcDE49KzUKnJm1WINF5nkSV6apqy4IkoEYzFYvIrSKSReDAevtRqluoHLTNIvJroJmqvns0KxYihz0bdR1Wkza3BdqKyEIRWSwivY9a7UKjpj9nRKQF0Ar431GoVyjVpM0PAdeKyFbgfQI9sSPGgiSgJjMVo6ojVTUD+CvwQMhrFVq/2GYRcQFPA38+ajUKrcOejboOq0mb3QSGt3oR+Ov8FRFJDnG9QqlGv8uOQcBUVfWFsD5HQ03aPBh4TVWbAn2A8c7v+BFhQRJw0JmKq5gE/DakNQq9g7U5AWgPzHdmWz4LmFmHv3Cv0WzUqlruLI4GOh+luoVKTf5fbwXeUdVKVd0A/EAgWOqqQ/ldHkTdH9aCmrX598AU2H+/Jw+BebiOCAuSgJrMVBz8y3UpsPYo1i8UfrHNqlqgqmmq2lIDsy0vBvqp6pLwVLfWajMbdV110DYDM4BzAUQkjcBQ1/qjWssjqyZtRkROBlKARUe5KPvr/QAAAzdJREFUfqFQkzZvBs4HEJF2BILkiN3xL5T3I6kztGYzFd8mIhcAlUA+cH34alx7NWzzcaOG7f252ajrpBq2+QPgIhFZCfiAe/TA+wTVKYfw/3owMEmd05jqshq2+c/AaBG5k8Cw19Aj2Xa7st0YY0yt2NCWMcaYWrEgMcYYUysWJMYYY2rFgsQYY0ytWJAYY4ypFQsSY2pJRB4SkbuPgXpsdK4FMeaosiAxxhhTKxYkxlRDROJE5D0R+V5ElovI1cF/8YtIFxGZH7TLaSLyPxFZKyLDnTKNROQT574Xy0XkN876F0VkiXP/j4eD3nOjiPxTRBY5208XkQ9EJEtEbnbK9HJec7pzD5GXqpszSUSuFZEvnfd+WUQiQvl5mRObBYkx1esNbFfV01S1PTDnIOU7Epg6pxvwoIg0BoYAH6hqJ+A04Dun7P2q2sXZp6eIdAx6nS2q2g34FHgNuILAPGePBJU5g8CVyh2ADGBAcEWcKTCuBro77+0DrjmEthtzSGyKFGOqtwx4UkT+Dbyrqp+KVDfJ6n7vqGopUCoiHxM42H8FjBWRSGCGqu4LkqtEJJPA718jAjcaWups2zeFxzIgXlX3AntFpCxoVt4vVXU9gIhMBM4BpgbV5XwCE05+5dQ5Bqjr9xkxxzALEmOqoaprRKQzgSm3R4jIXAJzcO3rxXuq7vLTl9BPRKQHgZ7KeBF5gkBP426gq6rmi8hrVV5r3+zD/qDn+5b3/b7+5L2qLAswTlX/dpBmGnNE2NCWMdVwhqZKVPUN4EngdGAjP04tP7DKLv1FxCMi9Qjc2+Mr58ZJ2ao6GhjjvEYiUAwUiEgD4JLDqN4ZzkyvLgJDWJ9V2f4RcIWI1HfakurUxZiQsB6JMdXrADwhIn4CMz7fQmCIaIyI3Ad8UaX8l8B7QHPgUVXdLiLXA/eISCVQBPxOVTeIyLfACgLTtS88jLotAv7l1PETYHrwRlVdKSIPAHOdsKkEbgU2HcZ7GXNQNvuvMXWIiPQC7lbVy8JdF2P2saEtY4wxtWI9EmOMMbViPRJjjDG1YkFijDGmVixIjDHG1IoFiTHGmFqxIDHGGFMrFiTGGGNq5f8BboUmFAIsW7sAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#取得的colsample_bytree和subsample分别为0.8和0.7，因为0.8和0.7在设定的值的边缘，所以不确定是不是最优解，把图像打出来观察曲线的变化\n",
    "\n",
    "print(\"Best: %f using %s\" % (gsearch4.best_score_, gsearch4.best_params_))\n",
    "test_means = gsearch4.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch4.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch4.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch4.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "\n",
    "pd.DataFrame(gsearch4.cv_results_).to_csv('my_preds_subsampleh_colsample_bytree_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(colsample_bytree), len(subsample))\n",
    "train_scores = np.array(train_means).reshape(len(colsample_bytree), len(subsample))\n",
    "\n",
    "for i, value in enumerate(colsample_bytree):\n",
    "    pyplot.plot(subsample, -test_scores[i], label= 'test_colsample_bytree:'   + str(value))\n",
    "\n",
    "\n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'subsample' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'subsample_vs_colsample_bytree1.png' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [0.5, 1, 1.5, 2], 'reg_lambda': [0.5, 1, 2]}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#发现曲线在subsample=0.7和colsample_bytree=0.8时是最优的，后续有上涨的形式，也可以把参数再往大调一些，确定是否为上涨的形式。\n",
    "\n",
    "#然后开始调整L1和L2，\n",
    "\n",
    "reg_alpha = [ 0.5,1,1.5, 2]    #default = 0, 测试0.1,1，1.5，2\n",
    "reg_lambda = [0.5, 1, 2]      #default = 1，测试0.1， 0.5， 1，2\n",
    "\n",
    "param_test5 = dict(reg_alpha=reg_alpha, reg_lambda=reg_lambda)\n",
    "param_test5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\AI\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58418, std: 0.00452, params: {'reg_alpha': 0.5, 'reg_lambda': 0.5},\n",
       "  mean: -0.58464, std: 0.00393, params: {'reg_alpha': 0.5, 'reg_lambda': 1},\n",
       "  mean: -0.58478, std: 0.00443, params: {'reg_alpha': 0.5, 'reg_lambda': 2},\n",
       "  mean: -0.58462, std: 0.00418, params: {'reg_alpha': 1, 'reg_lambda': 0.5},\n",
       "  mean: -0.58435, std: 0.00520, params: {'reg_alpha': 1, 'reg_lambda': 1},\n",
       "  mean: -0.58457, std: 0.00444, params: {'reg_alpha': 1, 'reg_lambda': 2},\n",
       "  mean: -0.58481, std: 0.00417, params: {'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  mean: -0.58400, std: 0.00440, params: {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  mean: -0.58470, std: 0.00466, params: {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  mean: -0.58484, std: 0.00455, params: {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  mean: -0.58500, std: 0.00423, params: {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  mean: -0.58486, std: 0.00417, params: {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       " -0.5839993047291475)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#设置其余值为之前获得的参数n_estimators=220；max_depth=6；min_child_weight=5；subsample=0.7；colsample_bytree=0.8\n",
    "xgb5 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=220,  \n",
    "        max_depth=6,\n",
    "        min_child_weight=5,\n",
    "        gamma=0,\n",
    "        subsample=0.7,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "gsearch5 = GridSearchCV(xgb5, param_grid = param_test5, scoring='neg_log_loss',n_jobs=-1, cv=5)\n",
    "gsearch5.fit(x_train , y_train)\n",
    "\n",
    "gsearch5.grid_scores_, gsearch5.best_params_,     gsearch5.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#得到最优的reg_alpha=1.5；reg_lambda=1，再次调整n_estimators的值\n",
    "def modelfit(alg, x_train, y_train, cv_folds=3, early_stopping_rounds=10):\n",
    "    xgb_param = alg.get_xgb_params()\n",
    "    xgb_param['num_class'] = 3\n",
    "    \n",
    "    xgtrain = xgb.DMatrix(x_train, label = y_train)\n",
    "        \n",
    "    cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], folds =cv_folds,\n",
    "             metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)\n",
    "  \n",
    "    cvresult.to_csv('6_nestimators.csv', index_label = 'n_estimators')\n",
    "    \n",
    "    #最佳参数n_estimators\n",
    "    n_estimators = cvresult.shape[0]\n",
    "    \n",
    "    # 采用交叉验证得到的最佳参数n_estimators，训练模型\n",
    "    alg.set_params(n_estimators = n_estimators)\n",
    "    alg.fit(x_train, y_train, eval_metric='mlogloss')\n",
    "        \n",
    "    train_predprob = alg.predict_proba(x_train)\n",
    "    logloss = log_loss(y_train, train_predprob)\n",
    "\n",
    "    print (\"logloss of train is:\", logloss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of train is: 0.47548789553557297\n"
     ]
    }
   ],
   "source": [
    "#把learning_rate设为0.02，n_estimators设为一个比较大的值其余参数保存之前获得的最优参数：max_depth=6；min_child_weight=5；subsample=0.7；colsample_bytree=0.8；reg_alpha=1.5；reg_lambda=1\n",
    "xgb6 = XGBClassifier(\n",
    "        learning_rate =0.02,\n",
    "        n_estimators=2000,  \n",
    "        max_depth=6,\n",
    "        min_child_weight=5,\n",
    "        gamma=0,\n",
    "        subsample=0.7,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        reg_alpha=1.5,\n",
    "        reg_lambda=1,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "modelfit(xgb6, x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'base_score': 0.5,\n",
       " 'booster': 'gbtree',\n",
       " 'colsample_bylevel': 0.7,\n",
       " 'colsample_bytree': 0.8,\n",
       " 'gamma': 0,\n",
       " 'learning_rate': 0.02,\n",
       " 'max_delta_step': 0,\n",
       " 'max_depth': 6,\n",
       " 'min_child_weight': 5,\n",
       " 'missing': None,\n",
       " 'n_estimators': 1226,\n",
       " 'nthread': 1,\n",
       " 'objective': 'multi:softprob',\n",
       " 'reg_alpha': 1.5,\n",
       " 'reg_lambda': 1,\n",
       " 'scale_pos_weight': 1,\n",
       " 'seed': 3,\n",
       " 'silent': 1,\n",
       " 'subsample': 0.7}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#查看各个参数的取值\n",
    "xgb6.get_xgb_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\AI\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:2: FutureWarning: from_csv is deprecated. Please use read_csv(...) instead. Note that some of the default arguments are different, so please refer to the documentation for from_csv when changing your function calls\n",
      "  \n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmcHGWdx/HPr3vOZI5MkknInQDhCKcYThUQBAEVPBBBcUVdUXfxdl1YXddlxfsWVkQX8AJEvABRVjlEVg0JNyEkhJyT+55MjsnM9G//eJ6e6cz0ZCZHT/VMf9+vV7266qmnq39VNdO/rqeqnjJ3R0REBCCVdAAiIlI8lBRERKSTkoKIiHRSUhARkU5KCiIi0klJQUREOikpiOQws38zsx8mHYdIUpQUBhkzqzGzJWb29pyyWjNbZmYX55TNNLN7zWyTmW02s+fN7Doza4jzrzCzDjNricMiM/tggWM/08yaCvkZeyNfPO7+BXf/xwJ93hIze00hll0IA7W/Btt2GeqUFAYZd28BrgS+bWaNsfgrwBx3vwvAzE4DHgb+DzjC3UcA5wHtwHE5i/ubu9e4ew1wMfAVM3vZwKyJ7A0zK0s6BikR7q5hEA7ArcDtwJnABmBczrxHge/28f4rgEe7lT0GvD1n+kJgLrCZkGSOzJl3ZCzbHOtcmDPvAuB5YCuwAvgkMBzYAWSAljiM72W9bgB+F98/CzikH9vjCOCPwEZgPnDJvsQDfA74aXzfVMCBdwPLgU3AB4ATgWfiul+f8zmHAA/G/bEe+BkwIs77SfysHfGzPtWPbbwE+Nf4Wa1AWZxeEddlPnB2nm1xCrAaSOeUvQl4Jo6fBMwBmoE1wDd62aZnAk29zKsHfgysA5YCnwFScV4a+HrcBouBq+J2LOtlWUuA1/Qy733Awrhf787+zQAGfBNYC2yJ2+jo3vZ30v+vg2lIPAAN+7jjoAFYFf/x3p1TPhzoAM7s4/1XkJMU4hfdZuCwOH0YsA04BygHPhX/OSvi9ELg3+L0WfEf8PD43lXAq3LiPCGO9/olkxPHrfEL4KT4Jfgz4I4+3jOc8KX97vieE+J2OWpv4yF/UrgRqALOBXYCvwHGABPil9IZsf6hcXtVAo3AI8C3cpa925ffnrZxTv2ngElANXB4XM/xOfHlTZjAS8A5OdO/AK6O438D3hnHa4BTellGr/uLkBB+C9TGOBYA743zPkD4Up4Yt/ef2IekEP+u1sf9WQl8F3gkznst8DgwgpAgjiT+MOptf2vo36Dmo0HK3TcRfmEOA36VM6uB0Cy4OltgZl+J5xW2mdlncuqeEstbCEcJPwFejPPeBvzO3f/o7m3A1whfTKcRfonWAF9y913u/iBwL3BZfG8bMMPM6tx9k7s/sZer9yt3f8zd2wlJ4fg+6r8eWOLut7h7e/y8XxKaxA5EPP/l7jvd/X8JX+K3u/tad18B/AV4GYC7L4zbq9Xd1wHfAM7Yw3L3tI2zvuPuy919ByHZV8Z1KXf3Je7+Ui/Lvp24P8yslvDr+fac7XGomY129xZ3//vebAwzS8fYr3H3re6+hHBk8M5Y5RLg2+7eFP9Ov7Q3y8/xDuBmd3/C3VuBa4BTzWxqXIdawhGiufs8d1+Vs377s79LmpLCIGVmlxN+of0J+HLOrE2EZopx2QJ3/5SH8wq/JvySzvq7u4/wcE7hIOAo4Atx3nhCs0B2GRnCr9QJcd7yWJa1NM4DeAvhS2ipmf3ZzE7dy9VbnTO+nZCA9mQKcHJMcJvNbDPhC+WgAxTPmpzxHXmmawDMbIyZ3WFmK8ysGfgpMHoPy93TNs5anjN/IfBRwtHM2vhZ43tZ9m3Am82sEngz8IS7Zz/rvYSjlBfMbLaZvX4PMeYzmnCEuDSnLHf/j8+Nu9v43ui+fVoITXMT4g+R6wlNjWvM7CYzq4tV93d/lzQlhUHIzMYQ2lPfB7wfuMTMTgdw922Edvg3780y3X0N4df1G2LRSsKXbfYzjdCMsSLOm2RmuX8/k+M83H22u19EaGL5DXBn9mP2Jqa9sBz4c0xw2aHG3T84wPF8MS7zWHevAy4nNG1kdf+8PW3jvO9x99vc/ZXxfc7uPwhy6z1P+EI9H3g7IUlk573o7pcRtseXgbvMbHj/V5P1hF/jU3LKOvc/oflmYs68SXux7Fzdt89wYBRdf2ffcfeXE37MHAb8SyzvbX9LPygpDE7XA79x94fiIfOngB/EX4XE6feY2dUxgWBmE4FpvS3QzEYRTkbOjUV3Aq8zs7PNrBz4BOFk518JSWcb8CkzKzezMwnJ5A4zqzCzd5hZfWwSaSY0e0D4hT3KzOoP0HbIuhc4zMzeGeMpN7MTzezIAY6nlnASebOZTSB+SeVYAxycM72nbdyDmR1uZmfF/byTcJTSka9udBvwYeB0wjmF7HIuN7PGeGSyORb3uhwzq8odCEeidwLXxcuhpwAfJxwZZdfrI2Y2wcxGEE6O96W82+eUxfjfbWbHx3X+AjDL3ZfE/Xty3G7b4vbo6GN/S38kfVJDw94NwBsJv6BGdCt/ALguZ/pk4D7CP/1m4DngOmBUnH8F4Z8le+XNWkKb85icZbyJcMJwC/Bn4onbOO+oWLYl1nlTLK8A/kBoxmoGZgOvzHnfzYQmgM30fvXR53Omz6SPk9Ox3uGEK5bWxeU/SDgXsVfxkP9Ec1lO/SZyTuITvgg/k7NNHo/b8ynCl3xTTt2LgGXxsz7Zj228hN1PTB9LOPezlXAy/t582zCn/mTCF/jvupX/NO7vFsKPgDf28v4z4/p3Hw4lnLv6adzey4HP0nX1URnhSHYD4eqjjxGOLKyXz1mS5zM+H+d9gHDSPLu+E2P52YQrjlroutKrpq/9raHvweIGFhEpCDM7H7jR3af0WVkSp+YjETmgzKzazC4ws7LYjPYfhIscZBDQkYIMCmb2KuD3+eZ5uHpKioSZDSM0hR1BOO/xO+Aj7t6caGDSL0oKIiLSSc1HIiLSadB1sjV69GifOnVq0mGIiAwqjz/++Hp3b+yr3qBLClOnTmXOnDlJhyEiMqiY2dK+a6n5SEREchQsKZjZzWa21sye62X+EWb2NzNrNbNPFioOERHpv0IeKdxKeLBLbzYSbsH/WgFjEBGRvVCwpODujxC++Hubv9bdZxNufxcRkSIwKM4pmNmVZjbHzOasW7cu6XBERIasQZEU3P0md5/p7jMbG/u8okpERPbRoEgKIiIyMEomKaxY8iKP/vp7NG/p9TSHiEjJK9jNa2Z2O6E/9tFm1kToKbEcwN1vNLODgDlAHZAxs48CMwrVadbaeY/yyqev5qXpJ1BXf3IhPkJEZNArWFLw8Li/Pc1fze6P7CuoitrwqNwdW3SiWkSkNyXTfFQ9Ipygbt26IeFIRESKV8kkhZqYFNpa1icciYhI8SqZpFDXEJJCZtumhCMRESleJZMUqobV0urlsENJQUSkNyWTFDCj2WpI71RSEBHpTekkBaAlVUf5ri1JhyEiUrRKKinsLKujsm1z0mGIiBStkkoKreX1VHdsTToMEZGiVVJJoa1yBLWZgtwwLSIyJJRUUshUNVDnLXgmk3QoIiJFqaSSglU3UGltNG/V0YKISD4llRRSw0YC0LJJ/R+JiORTUkmhvHYUAC2blRRERPIpqaRQVaeeUkVE9qSkksKw2Cnerhb1lCoikk9JJYWa+pAU2pUURETyKqmkUJvtKXW7HskpIpJPSSWFsqrh7KACU0+pIiJ5lVRSANhqtaR3qv8jEZF8Si4ptKTrqdil5iMRkXxKLinsKB/JsDY1H4mI5FNySWFX5UhqO9R8JCKST8GSgpndbGZrzey5XuabmX3HzBaa2TNmdkKhYsnVUT2aBt9CR8YH4uNERAaVQh4p3Aqct4f55wPT43Al8L0CxtLJahoZZq1s2qwmJBGR7gqWFNz9EWBPZ3QvAn7swd+BEWY2rlDxZKXrxgKwZf3KQn+UiMigk+Q5hQnA8pzppljWg5ldaWZzzGzOunX7129RVX1ICi0bV+/XckREhqIkk4LlKcvb0O/uN7n7THef2djYuF8fOnxkOBjZuWnVfi1HRGQoSjIpNAGTcqYnAgVv06kbHZLCw0/OK/RHiYgMOkkmhbuBf4hXIZ0CbHH3gv98rxsVksIpY/VIThGR7soKtWAzux04ExhtZk3AfwDlAO5+I3AfcAGwENgOvLtQsewWV3k1LQwjtU3PVBAR6a5gScHdL+tjvgP/XKjP35MtqXrKW9V9tohIdyV3RzPA9rIGqtX/kYhIDyWZFHZWjqKmXTeviYh0V5JJob1qFPWZLYQWLBERySrJpODDG2mgma07WpMORUSkqJRkUnh0lZE2Z8P6NUmHIiJSVEoyKbz2pGMA2LJ2eR81RURKS0kmhZrGcCP1tg1NCUciIlJcSjIpNIydDMCujSsSjkREpLiUZFIYNmoiAN6sTvFERHKVZFKgrJJNXsv6VUuTjkREpKgUrJuLYtdc0cgUa046DBGRolKaRwrA9opGatvUKZ6ISK6STQptw8YwKrOBjozuahYRySrZpEDtOEazhQ3N25KORESkaJRsUigbMZ60OevX6F4FEZGskk0K2ctSm9cuSzgSEZHiUbJJoW5MuIHtnkefSDgSEZHiUbJJoT4mhTMzjyUciYhI8SjZpJCuHUM7Kagfn3QoIiJFo2STAqk0W1INlG9bnXQkIiJFo3STArC1Yiw1rUoKIiJZJZ0UdgyfSGPHGto7MkmHIiJSFAqaFMzsPDObb2YLzezqPPOnmNkDZvaMmT1sZhMLGU93Xj+RcWxg1ebtA/mxIiJFq2BJwczSwA3A+cAM4DIzm9Gt2teAH7v7scC1wBcLFU8+FaOmUmEdrF2p3lJFRKCwRwonAQvdfZG77wLuAC7qVmcG8EAcfyjP/IKqHTsNgC2rXhrIjxURKVqFTAoTgNyHIDfFslxPA2+J428Cas1sVPcFmdmVZjbHzOasW3fgejZtmHAIADvXLzlgyxQRGcwKmRQsT1n3Lkk/CZxhZk8CZwArgPYeb3K/yd1nuvvMxsbGAxZgxaipYfmbl++5oohIiSjkQ3aagEk50xOBlbkV3H0l8GYAM6sB3uLuWwoY0+4qhtNsdVS0qFM8EREo7JHCbGC6mU0zswrgUuDu3ApmNtrMsjFcA9xcwHjy2lI5jpqdelaziAgUMCm4eztwFXA/MA+4093nmtm1ZnZhrHYmMN/MFgBjgesKFU9vdgwbr3sVRESigj6j2d3vA+7rVvbZnPG7gLsKGUNfnt5ay+tZz8pNO5g8eniSoYiIJK6k72gGOPH446i2XaxYqZPNIiIlnxRGjJ8OwI/vezjZQEREikDJJ4X6iUcAcM5YPatZRKTkk4I1TCWDkdq0KOlQREQSV/JJgbJKNlHP8M3zk45ERCRxSgpAS/1hNLKRXe26LFVESpuSAtC8ZSNTbTXLNqoLbREpbUoKQMNJlzLCtrF8xYqkQxERSZSSAjBiwuEAbGl6IeFIRESSpaQA1Iw7DIDWtS8mHImISLKUFADiZam2UZelikhpU1IAKK9iC3XUNC/EvfsjH0RESoeSQrR99DFMtdWs2rIz6VBERBKjpBClxh7JIbaS+as2Jx2KiEhilBSi+pWPUmltrFkyL+lQREQS02dSMLNDzKwyjp9pZh82sxGFD21gDbv4BgDmzP5rwpGIiCSnP0cKvwQ6zOxQ4H+AacBtBY0qCaPDvQrHVurRnCJSuvqTFDLx0ZpvAr7l7h8DxhU2rARU1rDZ6hnZspCOjK5AEpHS1J+k0GZmlwHvAu6NZeWFCyk581LTOcRWsmSDnq0gIqWpP0nh3cCpwHXuvtjMpgE/LWxYyTj0qJkcbCtZsHJT0qGIiCSirK8K7v488GEAM2sAat39S4UOLAn1TQ9RYe2sWPQ8HDcp6XBERAZcf64+etjM6sxsJPA0cIuZfaM/Czez88xsvpktNLOr88yfbGYPmdmTZvaMmV2w96tw4FRc/H0Ati1/NskwREQS05/mo3p3bwbeDNzi7i8HXtPXm8wsDdwAnA/MAC4zsxndqn0GuNPdXwZcCvz33gR/wDUeQTsp0uueU3cXIlKS+pMUysxsHHAJXSea++MkYKG7L3L3XcAdwEXd6jhQF8frgZV7sfwDr7yapTaJGSzWA3dEpCT1JylcC9wPvOTus83sYKA/fUxPAJbnTDfFslyfAy43sybgPuBD/VhuQY2szHB0aglPN21JOhQRkQHXZ1Jw91+4+7Hu/sE4vcjd39KPZVu+xXWbvgy41d0nAhcAPzGzHjGZ2ZVmNsfM5qxbt64fH73v6s74Z8bYZhYvWljQzxERKUb9OdE80cx+bWZrzWyNmf3SzCb2Y9lNQO4lPBPp2Tz0XuBOAHf/G1AFjO6+IHe/yd1nuvvMxsbGfnz0vkuPPx6A7cueKOjniIgUo/40H90C3A2MJzT/3BPL+jIbmG5m08ysgnAi+e5udZYBZwOY2ZGEpFDYQ4G+HHQ0jjF8w1zaOjKJhiIiMtD6kxQa3f0Wd2+Pw61Anz/XY9cYVxHOR8wjXGU018yuNbMLY7VPAO8zs6eB24ErPOnLfipr2VYzlSP8JeaubE40FBGRgdbnzWvAejO7nPClDeE8wIb+LNzd7yOcQM4t+2zO+PPAK/oX6sBJd+zgmNRifrdkI8dPGnIdwoqI9Ko/RwrvIVyOuhpYBVxM6PpiyKquGsY428hdD/496VBERAZUf64+WubuF7p7o7uPcfc3Em5kG7reejMAx+56UjexiUhJ2dcnr338gEZRbMYezU4qOMKWsWi9ekwVkdKxr0kh3z0IQ0e6HB//ck5ILeC9t85OOhoRkQGzr0lhyLepVB18KkfZUnZub0k6FBGRAdPr1UdmtpX8X/4GVBcsoiJhk06i3Do4whfS3pGhLL2v+VNEZPDoNSm4e+1ABlJ0Jp2MYxzT9hxPLd/MzKkjk45IRKTg9PO3N8NGkhlzNK9Iz+WRBcneZC0iMlCUFPYgfeireXnqRf6+oCnpUEREBoSSwp4cfAbltFO9chYbWlqTjkZEpOCUFPZk8qlkUuWcmprLxTf+LeloREQKrj9dZ281s+Zuw/LYnfbBAxFkYiqGY+XVvDr9DFNGDUs6GhGRgutPh3jfIDwH4TbC5aiXAgcB84GbgTMLFVwxsKoRHNa6jOcXLmLLjpdRX12edEgiIgXTn+aj89z9++6+1d2b3f0m4AJ3/znQUOD4knfJjzDglf4kb/7v/0s6GhGRgupPUsiY2SVmlorDJTnzhvydzYw7Hk9XcEH6MTZu25V0NCIiBdWfpPAO4J3A2ji8E7jczKoJD9EZ2lIprHokp6efpa11B5u3KzGIyNDVn66zF7n7G9x9dBze4O4L3X2Huz86EEEm7sLvUkEbL/e5XHSDmpBEZOjqz9VHE+OVRmvNbI2Z/dLMJg5EcEVj2ulgKS4pf5S6Kp1oFpGhqz/NR7cAdwPjgQnAPbGsdJRXQXUDp/MEc1ds4oXVenaziAxN/UkKje5+i7u3x+FWoLHAcRWfYY3U2g5OSz3Hu25+LOloREQKoj9JYb2ZXW5m6ThcDmwodGBF5/0Pg6W5qup+trV2sHVnW9IRiYgccP1JCu8BLgFWA6uAi4F3FzKoolReDcNGcaI/x67WHbzuO6Vxjl1ESkt/rj5a5u4Xunuju49x9zcCb+7Pws3sPDObb2YLzezqPPO/aWZPxWGBmW3eh3UYOG/6Hmlv451lD7CmeSftHZmkIxIROaD2tUO8j/dVwczSwA3A+cAM4DIzm5Fbx90/5u7Hu/vxwHeBX+1jPANj2pmQKuf91Q/S2p7h3G8+knREIiIH1L4mBetHnZOAhfE+h13AHcBFe6h/GXD7PsYzMNJlUDOWxrYmDqnYxIrNO2jT0YKIDCH7mhT6073FBGB5znRTLOvBzKYA04AHe5l/pZnNMbM569Yl/BS0d9+HAT9MfVFHCyIy5PTaS6qZbSX/l78B1f1Ydr6jid6SyaXAXe7ekW9m7ITvJoCZM2cm299SwxSobmBq6xrqK5wVm3ews62DqvJ0omGJiBwIvR4puHutu9flGWrdvT9dbjcBk3KmJxK64M7nUoq96SjXG2/EMm3cWX89u9ozvObrf046IhGRA6KQT16bDUw3s2lmVkH44r+7eyUzO5zQBffgebTZ9HPBUhzeMouyFDRt3sHyjduTjkpEZL8VLCm4ezuhF9X7gXnAne4+18yuNbMLc6peBtzh7oOnG+5UCkYeDJ5h1kFfJ2Xwhut134KIDH42mL6LIZxTmDNnTtJhQHsrfH4sWIpXVd3F8k07OKRxOA984sykIxMR6cHMHnf3mX3VK2Tz0dBWVgkNU8E7eLj2s6QMXlq3jRWbdyQdmYjIPlNS2B//9DfASK+by0OfOAOAV335QXa1694FERmclBT2R3k1jDoUPMOUO87mhrefQMbhqP/4Q9KRiYjsEyWF/fXPs6B8GGxewutmjGJcfRVtHc6ZX30o6chERPaaksL+SqXhkp9A+0744kT+8qlXk04ZSzZs55xv6P4FERlclBQOhOmvgVQZdLRStmUJz33utQC8uLaFh+evTTg4EZH+U1I4UD76bHj97kyqy+DJfz8HgCtumc3vn12VYGAiIv2npHCg1I2HUdPBO+C6cTQMr+Dp/zgXgA/+7Aluf2xZwgGKiPRNSeFAump2ZzMSq56hvrqc568NTUnX/OpZTrruT7S25+3zT0SkKCgpHEhm8IkFYfz7r4IdmxlWUcbC686nIm2s3drKkf/+B5ZtUD9JIlKclBQOtOGj4Ir7wviXp0Cmg7J0igXXXcD33hHuYzj9qw/xs1lLGWxdjIjI0KekUAhTXwFlVWH8W8d0Fp9/zDj+evVZAHz6189x6Kd/z4I1W5OIUEQkLyWFQvnMGkiVQ/MKePzWzuLxI6pZ/MULmDpqGB0Z59xvPsJJ1/2JdVtbk4tVRCRSUiikT8dLUe/5CDz3y85iM+Phf3k1T/77OZTHcw0nXvcnvnr/C6zdujOhYEVE1HV24e3aDl8YF8bf9lM48g09qixa18K533yE9kzYF421ldxx5Skc0lgzkJGKyBDW366zlRQGws5m+FJ8MulF/w0ve0feaovXb+Pcb/6Zto6wT9Ip4zuXvoxzZoylokwHdSKy75QUik1rC3xxQhhvmAYfearXquu2tnLbrGV8808LOsvG1FbyjUuO55SDR1KWVoIQkb2jpFCM2lvh82PCeM04+NizkC7vtXpHxnnkxXW870dzOpuWAEbXVPCfFx7NK6ePpr669/eLiGQpKRSrTAdcdxB07AJLwyfmQ01jn2/b2dbBw/PXcdVtT+yWIAA+ee5hnHboaI6ZUE+5jiJEJA8lhWL33RNhQ2wees/9MPmUfr+1vSPDU8s38/D8dVz/0MIe8z989nROmjqSYyfVU1elIwkRUVIYHFY+CTedGcbrJ8GHn9xjc1Jv1re0MnvxRj53z1zWNOe/3+FDZx3KkePqOOKgWqaMGk46ZfsRuIgMNkoKg8XOZvjKwZBpC9P/+ABM7HO/7VHzzjaeXLaZ51Zs4av3z++1XmNtJVe9OiSLw8fWUj9MRxUiQ1VRJAUzOw/4NpAGfujuX8pT5xLgc4ADT7v72/e0zCGXFLLm3Qs/j5eq1oyDq2ZBVf0BW/zOtg5eXNPCvNXN/Otdz7CnvV6WMq44bSqTRg5j0shqJjUMY2LDMKor0gcsHhEZWIknBTNLAwuAc4AmYDZwmbs/n1NnOnAncJa7bzKzMe6+x0eVDdmkANC6NRw1dOwK02/4Nhx/OaTLCvJx7s7q5p28sGorL67dylfvn995j8SelKWM959xMAfVVdFYWxmGmjCuxCFSnIohKZwKfM7dXxunrwFw9y/m1PkKsMDdf9jf5Q7ppJC14gn4wau7pi+9DQ6/IHTNPUDcnXVbW1m+aTvLN+5g+cbtfP2PC/p+Y46ylHH5KVO6EkdtJY01lYyprWTk8ArdbyEygIohKVwMnOfu/xin3wmc7O5X5dT5DeFo4hWEJqbPufsf8izrSuBKgMmTJ7986dKlBYm5qLjDC/fCzy8P05aGd90NU14xoMmhN+0dGTZu38W6ra1dQ0srX/lD7+cw+jKuvop3njqF+upy6qrKw2t1eK2vLqe2qkyX3Irso2JICm8FXtstKZzk7h/KqXMv0AZcAkwE/gIc7e6be1tuSRwp5Opogyd+DL/7eFfZJT+GI14PqcHRVLNjVwfrW1pZm5M8/uueuezqR1NVfx1UV8nbTpzcmTxqKssYVllGTWWa4ZVlDK/IlqWpLBsc203kQOpvUihMY3XQBEzKmZ4IrMxT5+/u3gYsNrP5wHTC+QeBcInqie+F4y6Dp2+D330C7vyHMO9134DjLoWK4cnG2IfqinQ8aT2ss+ydp0zpUc/d2dmWoXlnG1t2hKE55/ULv3+BXe2ZvJ+xurmVbz/w4gGN24D3n3EI1eVpqspTVJWnqS5PU1meimXpzrLs/KqccR3VyGBUyCOFMkLT0NnACsIX/dvdfW5OnfMIJ5/fZWajgSeB4919Q2/LLbkjhe4yHTDvHvjFu7rKasfB5b+EsUclF1cRaO/I0NLaTvOOdrbtamdbazstre1s39VBS2uY/uIeEkux+MjZ06koS1GRTlGeNsrLUpTH8bJU12s6bZSnUpSlrbMsjKcoS3VNl+1WL8xLpwwrgmZIGTiJNx/FIC4AvkU4X3Czu19nZtcCc9z9bgt/lV8HzgM6gOvc/Y49LbPkk0KWOyz7G/zowq57HABe/y046o1Q3ZBcbENMR8Zpbe9gx64OdrZn2NkWxlvbO9jZlonlcbytgx888hLLNu5IOuyi8JGzp5OOSShlRjoFKcuOG6mUke5WnorjZkbKsuXE6a4yMzqXa5bz/s66XWWd81N7Xl6+ZWQTaN7l5by32JNsUSSFQlBSyGP7Rnj6drj/37rKho2CN3wHpp8LZRXJxSZ7zd1p63DaOjLsas+E144M7R1OeyZDW4fT3uG0ZTJ0ZEK97vO6xjO0ZZzP3/s8rUV+hCR9qyhLseDz5+/Te5UUSpE7rHoabr8Mtuacvpn5Xjj2bTDBIVg2AAAQLklEQVTppKK4cklKh7vTkXE63MlkCK/uZDJ5yjOOO2G+OxkP78/klGXnd2T2PD9b5nH5fS1vt/rZskxu3ex7c+uyW53dlw3/8+iift33szeqylO88F9KCrtRUuinjnZY9BDcfilk2rvKT/sQHHkRTHg5pHQiVKRUKClIl9at4eT0b68C7+gqP/F9MONCmHxawe6aFpHioKQg+e3YDC/+L/zmg7sfQRz3djj0bDjkLBg2Mrn4RKQglBSkb7u2wcIH4N6Pw/Z1XeWWhjP+FQ59DYw/ftDcJCcivVNSkL2T6Qh9Li38E/y5W2e2x7w1JIhDzoKaMcnEJyL7RUlB9s+2DeFE9Yt/hGe63Tpy4vtg2qtg6qvU1CQySCgpyIGTycDqp2HRw/DAf+1+shrglH+CaafD5FOhekQiIYrInikpSOG074KVT8DiR+DhL/VMEqdeBVNOg4knQU1jMjGKyG6UFGTgtO2Eptmw9P/gz1/pliQMjr0EJp4Ybp4bc5QufxVJgJKCJKdtJ6x6CpbPguWPhedC5Jr6qpAgJp0ckoXOS4gUXDF0nS2lqrwKJp8SBgjdb2xeCstnQ9Nj8NhNsOQvXfUtFboGzx5NNB6hy2BFEqIjBUnGrm2w8slwNPHX62HHxt3nV42Ak98fuuMY/zJdCiuyn9R8JIOLO2xcFJqbmh6DOTf3rHPkG0KCGH9CeNWVTiL9pqQgg19rS+j1deWT4Wqnub8Gz+n+uawKDr8Axh3XNej8hEheSgoyNO3YBCufCkli5ZMw714g5284XQkHnxmeQjf2KBh7NIw6VFc8ScnTiWYZmqob4JBXhyFr+8ZwRLHqaVgzN1zt9OL9u79v3HEhQeQmi+GjBzZ2kUFASUEGv2EjeyaK9l2wfgGseS4Oc+Gpn+3+vpqxuyeJsUfB6MOgrHJg4xcpIkoKMjSVVcBBR4chV8s6WDs3JIk1c8NjTF96cPc6Y2bkJItjwmvtQXpqnZQEnVMQ6WiHjS/B6me7kkX35icMpr5y9yaoxsOhYngiIYvsLZ1TEOmvdFn4gm88HI65uKt8xyZY83xMFLEJatb3ur3ZYPq5MOaIcNNd4xFKFjKoKSmI9Ka6Aaa+IgxZmQ7YtCQkibUvwLoXYMEfeh5ZZK+CGnMENB4JjYeF8xWVtQO4AiJ7r6BJwczOA74NpIEfuvuXus2/AvgqsCIWXe/uPyxkTCL7JZWGUYeEYcZFXeUd7bBpMaydFxLF2nm9JIsKmPKKcDQxejqMmh5ea8fpnIUUhYIlBTNLAzcA5wBNwGwzu9vdn+9W9efuflWh4hAZEOmy8OU+ejpwYVd5RxtsXAzr58O6+eGKqHXzYdaNPZcx7rhwT8WoQ0OyGHVIGK+qG7DVECnkkcJJwEJ3XwRgZncAFwHdk4LI0JUuD01HjYeFbjqyMhnYuhLWvwgbFna9PvcrdrsZD8Kls9kkMXp6V9JomBKWL3IAFTIpTACW50w3ASfnqfcWMzsdWAB8zN2Xd69gZlcCVwJMnjy5AKGKDLBUCuonhiH3/goIXY9vWpyTLF6Ceb+FpY/2XE7nkUUcskmjZqyao2SfFDIp5PuL7H796z3A7e7eamYfAH4EnNXjTe43ATdBuCT1QAcqUlTKq2DMkWHIeuMN4XX7xpAkNuQcYbxwbzh/kcvScNAxOUcWOUNlzcCtiww6hUwKTcCknOmJwMrcCu6+IWfyB8CXCxiPyOA3bGQYJp24e3kmA81NXUcW2aTx7C96LqN2XP6jixFT1EeUFDQpzAamm9k0wtVFlwJvz61gZuPcfVWcvBCYV8B4RIauVApGTA7DoWfvPq9tRzjZ3Xl0sTC8Pn5Lz+Vkr4YaeTA0TIWR08J4/SSdvygRBUsK7t5uZlcB9xMuSb3Z3eea2bXAHHe/G/iwmV0ItAMbgSsKFY9IySqvhrEzwtDd9o1dJ7lzk8b8+3rWbZgak8W0rmTRMC2UVwwr9FrIAFE3FyLSUyYDLWvCg482LQ5HGtnx1c9Cpn33+ukKmDAzJotpMGJquDpqxJTw1Dyd9E6curkQkX2XSkHduDDk3tGdtX1jV7LoTBqLQ+eCT63qWX/0YSFBNEyJzVxTupJGdYOSRhFRUhCRvZc94T3h5T3n7doOm5fFYWnoFiQ7vvCPPetX1HYliB5JY7K6BhlgSgoicmBVDAt9Po05Iv/8nVtCkti0NCaNpXF6Mcz/Xc/61SO7ksaIyXF8ateJ9fKqgq5OqVFSEJGBVVUf7qE46Jie89xh+4acZJGTNNY8B8//pud7ag7Kf4QxYkq4OVBXTe0VJQURKR5m4TGpw0fnb5rKngDPTRrZ8eWzet6XYSmom5hzhNEtadSOC+dPpJOSgogMHrknwCef0nN+Rzs0r+iWNGJT1bO/gI5d3d5g8WqpnGRRPyl2QTIpJI0Su6GvtNZWRIa2dFn4cm+YAtPyzG/bCVuaYPOSrmapbAKZd09ouuqubiKMmNTVV1X9pK7EMWLSkDsRrqQgIqWjvApGHxqGfHZtgy0rYMvyODTFJLIclj8Gz95Fjy7cKuuhfgLUTYivE8Nr/cRQVjdhUJ0MV1IQEcmqGN7V1Xk+mQ5oWduVNDYvD81VW1aEvqdWPgnb1/d83/DGriRRn3vUEYfhY4rm3IaSgohIf6XSXec0Jp2Uv07bzpAomlfEI42YMLY0hW5EFj0Mu1q6Lbcc6sZ3Sxg5CaRuwoA9bElJQUTkQCqv6npkaz7u4V6NbNNUNmFkh6V/hS3Ler6vsh5e8SE4/V8KGr6SgojIQDKD6hFhOOjo/HUyHbB1dTzayDm30djLDYEHkJKCiEixSaVj89GE3pupCvXRA/ppIiJS1JQURESkk5KCiIh0UlIQEZFOSgoiItJJSUFERDopKYiISCclBRER6WTu3netImJm64Cl+/j20UCe3qoGHa1H8RgK6wBaj2JTiPWY4u6NfVUadElhf5jZHHefmXQc+0vrUTyGwjqA1qPYJLkeaj4SEZFOSgoiItKp1JLCTUkHcIBoPYrHUFgH0HoUm8TWo6TOKYiIyJ6V2pGCiIjsgZKCiIh0KpmkYGbnmdl8M1toZlcnHU9vzGySmT1kZvPMbK6ZfSSWjzSzP5rZi/G1IZabmX0nrtczZnZCsmuwOzNLm9mTZnZvnJ5mZrPievzczCpieWWcXhjnT00y7lxmNsLM7jKzF+J+OXWw7Q8z+1j8e3rOzG43s6rBsC/M7GYzW2tmz+WU7fW2N7N3xfovmtm7imQ9vhr/pp4xs1+b2YicedfE9ZhvZq/NKS/895i7D/kBSAMvAQcDFcDTwIyk4+ol1nHACXG8FlgAzAC+Alwdy68GvhzHLwB+DxhwCjAr6XXotj4fB24D7o3TdwKXxvEbgQ/G8X8CbozjlwI/Tzr2nHX4EfCPcbwCGDGY9gcwAVgMVOfsgysGw74ATgdOAJ7LKdurbQ+MBBbF14Y43lAE63EuUBbHv5yzHjPid1QlMC1+d6UH6nss0T/WAdwhpwL350xfA1yTdFz9jP23wDnAfGBcLBsHzI/j3wcuy6nfWS/pAZgIPACcBdwb/1nX5/wjdO4X4H7g1DheFutZEaxDXfxCtW7lg2Z/xKSwPH4plsV98drBsi+Aqd2+TPdq2wOXAd/PKd+tXlLr0W3em4CfxfHdvp+y+2OgvsdKpfko+0+R1RTLilo8bH8ZMAsY6+6rAOLrmFitmNftW8CngEycHgVsdvf2OJ0ba+d6xPlbYv2kHQysA26JzWA/NLPhDKL94e4rgK8By4BVhG37OINvX2Tt7bYvun2Sx3sIRzmQ8HqUSlKwPGVFfS2umdUAvwQ+6u7Ne6qapyzxdTOz1wNr3f3x3OI8Vb0f85JURjjs/567vwzYRmiy6E3RrUdsc7+I0BQxHhgOnJ+narHvi770FndRr4+ZfRpoB36WLcpTbcDWo1SSQhMwKWd6IrAyoVj6ZGblhITwM3f/VSxeY2bj4vxxwNpYXqzr9grgQjNbAtxBaEL6FjDCzMpindxYO9cjzq8HNg5kwL1oAprcfVacvouQJAbT/ngNsNjd17l7G/Ar4DQG377I2tttX4z7BAgnwIHXA+/w2CZEwutRKklhNjA9Xm1RQTh5dnfCMeVlZgb8DzDP3b+RM+tuIHvVxLsI5xqy5f8Qr7w4BdiSPbROkrtf4+4T3X0qYXs/6O7vAB4CLo7Vuq9Hdv0ujvUT/zXn7quB5WZ2eCw6G3iewbU/lgGnmNmw+PeVXYdBtS9y7O22vx8418wa4lHTubEsUWZ2HvCvwIXuvj1n1t3ApfEqsGnAdOAxBup7bKBPtiQ1EK5MWEA4e//ppOPZQ5yvJBwSPgM8FYcLCG26DwAvxteRsb4BN8T1ehaYmfQ65FmnM+m6+ujg+Ae+EPgFUBnLq+L0wjj/4KTjzon/eGBO3Ce/IVzBMqj2B/CfwAvAc8BPCFe2FP2+AG4nnAdpI/xSfu++bHtCm/3COLy7SNZjIeEcQfb//Mac+p+O6zEfOD+nvODfY+rmQkREOpVK85GIiPSDkoKIiHRSUhARkU5KCiIi0klJQUREOikpiIhIJyUFkX4ws+PN7IKc6QsPVNfFZvZRMxt2IJYlsr90n4JIP5jZFYSboa4qwLKXxGWv34v3pN2940DHIqIjBRlSzGyqhQfh/CA+VOZ/zay6l7qHmNkfzOxxM/uLmR0Ry98aH0bztJk9ErsUuBZ4m5k9ZWZvM7MrzOz6WP9WM/uehYcjLTKzM+JDVeaZ2a05n/c9M5sT4/rPWPZhQid1D5nZQ7HsMjN7Nsbw5Zz3t5jZtWY2CzjVzL5kZs/Hh7R8rTBbVEpO0rffa9BwIAdCn/XtwPFx+k7g8l7qPgBMj+MnE/r4gdBFwoQ4PiK+XgFcn/PezmngVkKnf0bojbQZOIbwo+vxnFiy3TGkgYeBY+P0EmB0HB9P6KuokdBD64PAG+M8By7JLovQBYLlxqlBw/4OOlKQoWixuz8Vxx8nJIrdxK7JTwN+YWZPER68Mi7O/j/gVjN7H+ELvD/ucXcnJJQ17v6su2eAuTmff4mZPQE8CRxFeMJWdycCD3vo0TTbnfLpcV4HofdcCIlnJ/BDM3szsL3HkkT2QVnfVUQGndac8Q4gX/NRivCQmeO7z3D3D5jZycDrgKfMrEedPXxmptvnZ4Cy2NvlJ4ET3X1TbFaqyrOcfH3mZ+30eB7B3dvN7CRCj6eXAlcRuicX2S86UpCS5OHBRYvN7K3Q+dD34+L4Ie4+y90/S3gU5SRgK+GZ2fuqjvCAni1mNpbdH3KTu+xZwBlmNtrM0oRHSf65+8LikU69u98HfJTQk6vIftORgpSydwDfM7PPAOWE8wJPA181s+mEX+0PxLJlwNWxqemLe/tB7v60mT1JaE5aRGiiyroJ+L2ZrXL3V5vZNYRnHRhwn7v/tucSqQV+a2ZVsd7H9jYmkXx0SaqIiHRS85GIiHRS85EMeWZ2A+GZ0bm+7e63JBGPSDFT85GIiHRS85GIiHRSUhARkU5KCiIi0klJQUREOv0/Iz8qengfpNwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#打印出来图像，看一下曲线的变化。\n",
    "cvresult = pd.DataFrame.from_csv('6_nestimators.csv')\n",
    "        \n",
    "test_means = cvresult['test-mlogloss-mean']\n",
    "test_stds = cvresult['test-mlogloss-std'] \n",
    "        \n",
    "train_means = cvresult['train-mlogloss-mean']\n",
    "train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "x_axis = range(0, cvresult.shape[0])\n",
    "        \n",
    "pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "pyplot.xlabel( 'n_estimators' )\n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'n_estimators6.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#把调整好的模型进行保存\n",
    "import pickle\n",
    "pickle.dump(xgb6, open(\"xgb_model.pkl\", 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of train is: 0.47548789553557297\n"
     ]
    }
   ],
   "source": [
    "#把数据进行保存\n",
    "#最终的参数为learning_rate =0.02；n_estimators=1226；max_depth=6；min_child_weight=5；subsample=0.7；colsample_bytree=0.8；reg_alpha=1.5；reg_lambda=1\n",
    "import pickle\n",
    "\n",
    "xgb = pickle.load(open(\"xgb_model.pkl\", 'rb'))\n",
    "\n",
    "train_predprob = xgb.predict_proba(x_train)\n",
    "logloss = log_loss(y_train, train_predprob)\n",
    "\n",
    "print (\"logloss of train is:\", logloss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
